diff --git a/AGENTS.md b/AGENTS.md index 56988faf..351b9617 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,7 @@ python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_skill_interpretation.py . python3 scripts/render_review_viewer.py . +python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT" python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT" ``` @@ -62,6 +63,7 @@ python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED python3 scripts/render_skill_overview.py . python3 scripts/render_skill_interpretation.py . python3 scripts/render_review_viewer.py . +python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT" python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT" ``` @@ -96,6 +98,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/world_class_evidence_contract.py`: shared intake contract and artifact-integrity validator. Keep ledger, intake, and submission review aligned so source evidence cannot be accepted without a valid real submission and matching artifact SHA-256 checks. - `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state. - `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness. +- `scripts/render_world_class_preflight.py`: operator-facing collection preflight for world-class evidence. Keep environment and external prerequisite checks redacted, and never let preflight count as accepted evidence. - `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. diff --git a/Makefile b/Makefile index f89e090c..0250b905 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -115,6 +115,9 @@ world-class-ledger-check: world-class-intake-check: $(PYTHON) tests/verify_world_class_evidence_intake.py +world-class-preflight-check: + $(PYTHON) tests/verify_world_class_preflight.py + world-class-submission-review-check: $(PYTHON) tests/verify_world_class_submission_review.py @@ -247,7 +250,7 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py diff --git a/README.md b/README.md index 18e56a5d..90fa9bf1 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks - a world-class evidence ledger that records which external and human evidence is accepted or still pending without treating planned work as proof - a world-class evidence intake contract that validates external and human evidence packets for provenance, privacy, artifact refs, and anti-overclaim rules before ledger review +- a redacted world-class preflight report that checks local files, environment readiness, human/external prerequisites, and source blockers before operators collect evidence - a world-class submission review queue that compares evidence packets, intake validation, source artifacts, and ledger state without accepting evidence - a world-class operator runbook that gives reviewers the exact commands, artifacts, and collection checklist needed to close remaining evidence gaps - a world-class claim guard that scans public claim surfaces and blocks premature completed/true claims while the evidence ledger still has pending external or human evidence @@ -151,6 +152,7 @@ python3 scripts/yao.py install-simulate . --package-dir dist python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json python3 scripts/yao.py world-class-evidence . SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}" +python3 scripts/yao.py world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-intake . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR" @@ -428,7 +430,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-report, skill-interpretation, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-submission-kit, world-class-submission-review, world-class-runbook, world-class-claim-guard, benchmark-reproducibility, evidence-consistency, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-report, skill-interpretation, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-preflight, world-class-submission-kit, world-class-submission-review, world-class-runbook, world-class-claim-guard, benchmark-reproducibility, evidence-consistency, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -442,6 +444,7 @@ Utility scripts that make the meta-skill operational: - `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence - `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts - `render_world_class_evidence_intake.py`: validates world-class external and human evidence packets against provenance, privacy, artifact, and anti-overclaim requirements before ledger review +- `render_world_class_preflight.py`: renders redacted collection preflight checks for pending provider, human, native-permission, and native-client evidence without accepting evidence - `render_world_class_submission_review.py`: renders a read-only queue that compares submissions, intake validation, source evidence, and ledger state without accepting evidence - `render_world_class_operator_runbook.py`: renders an operator-facing checklist and command map for collecting pending world-class evidence without accepting evidence - `render_world_class_claim_guard.py`: scans README, docs, and reports for premature world-class completion claims while accepted evidence is still pending diff --git a/evidence/world_class/README.md b/evidence/world_class/README.md index 91c6951f..337df467 100644 --- a/evidence/world_class/README.md +++ b/evidence/world_class/README.md @@ -8,6 +8,7 @@ Run: ```bash SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}" +python3 scripts/yao.py world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-intake . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR" @@ -23,6 +24,8 @@ The intake validator checks: - credentials, secrets, raw user content, and raw provider prompts are explicitly excluded - planned work, local command-only output, and metadata fallback are not claimed as completion evidence +Run `world-class-preflight` before assigning external or human work. It checks local files, redacted environment readiness, human/external prerequisites, and source-evidence blockers without accepting evidence or printing secrets. + The generated intake report also includes an `operator_checklist` for each pending evidence item. Use it to find the template path, target submission path, preparation command, validation command, required provenance, success checks, and privacy boundary before asking a reviewer or external operator to submit evidence. The submission kit command creates editable JSON drafts plus a local README for an external operator or human reviewer. Those drafts keep `template_only: true` and do not count as evidence until the real run or review exists, the packet is edited truthfully, every artifact ref points to a local aggregate evidence file with a matching `sha256`, and `world-class-intake` validates it. diff --git a/registry/index.json b/registry/index.json index 3d768e41..c1ec5bc2 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544" + "package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index c4a865a6..deaa408a 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544", - "archive_sha256": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf" + "package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1", + "archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf", + "archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 8bebba9e..49da2bb7 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T17:08:05Z", + "generated_at": "2026-06-15T17:38:29Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index d1d9c529..e8e1bc12 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,12 +4,12 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 174, - "script_file_count": 111, - "test_file_count": 63, - "internal_module_count": 32, - "cli_script_count": 81, - "command_handler_count": 64, + "python_file_count": 177, + "script_file_count": 113, + "test_file_count": 64, + "internal_module_count": 33, + "cli_script_count": 82, + "command_handler_count": 65, "entrypoint_command_handler_count": 18, "command_module_count": 6, "warn_line_threshold": 900, @@ -31,7 +31,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 810, + "lines": 821, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -85,6 +85,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 707, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/apply_adaptation.py", "lines": 706, @@ -98,13 +105,6 @@ "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, - { - "path": "scripts/render_review_viewer.py", - "lines": 685, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Split viewer data assembly from HTML section rendering." } ], "watchlist": [ @@ -117,7 +117,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 810, + "lines": 821, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 45f3de88..2fceb410 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,12 +5,12 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `174` -- scripts: `111` -- tests: `63` -- internal modules: `32` -- CLI scripts: `81` -- Yao CLI command handlers: `64` +- python files: `177` +- scripts: `113` +- tests: `64` +- internal modules: `33` +- CLI scripts: `82` +- Yao CLI command handlers: `65` - entrypoint command handlers: `18` - command modules: `6` - largest file lines: `899` @@ -30,7 +30,7 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `scripts/yao_cli_parser.py` | `810` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `scripts/yao_cli_parser.py` | `821` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `801` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_review_studio.py` | `763` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | @@ -43,7 +43,7 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | -| `scripts/yao_cli_parser.py` | `810` | `internal-module` | `pass` | +| `scripts/yao_cli_parser.py` | `821` | `internal-module` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | | `scripts/skill_report_model.py` | `801` | `internal-module` | `pass` | | `tests/verify_review_studio.py` | `763` | `test` | `pass` | @@ -51,9 +51,9 @@ No file-size hotspots found. | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | +| `scripts/render_evidence_consistency.py` | `707` | `cli-script` | `pass` | | `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` | | `scripts/review_studio_layout.py` | `694` | `internal-module` | `pass` | -| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/context_budget.json b/reports/context_budget.json index 469ba511..9fb71409 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -1,21 +1,23 @@ { "ok": true, "failures": [], - "warnings": [], + "warnings": [ + "Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." + ], "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1377236, + "other_text_tokens": 1395152, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1378003, - "deferred_resource_tokens": 436837, + "estimated_total_text_tokens": 1395919, + "deferred_resource_tokens": 442670, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387626, - "file_count": 111 + "estimated_tokens": 393459, + "file_count": 113 }, { "path": "references", @@ -31,37 +33,38 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387626, - "file_count": 111 + "estimated_tokens": 393459, + "file_count": 113 } ], "deferred_resource_governance": { - "status": "governed", + "status": "needs-review", "large_dir_count": 1, - "governed_large_dir_count": 1, + "governed_large_dir_count": 0, "directories": [ { - "status": "governed", + "status": "needs-review", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ - "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [], + "missing": [ + "trust report covers scripts" + ], "path": "scripts", - "estimated_tokens": 387626, - "file_count": 111, + "estimated_tokens": 393459, + "file_count": 113, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "Large deferred resources are indexed and backed by evidence." + "summary": "One or more large deferred resource directories still need explicit governance evidence." }, - "relevant_file_count": 565, + "relevant_file_count": 573, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 diff --git a/reports/context_budget.md b/reports/context_budget.md index ee8b248b..0f44c15c 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 436837 | `governed` | scripts:387626 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 442670 | `needs-review` | scripts:393459 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 88e78c34..2c8bec4d 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,44 +8,47 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 436837, + "deferred_resource_tokens": 442670, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387626, - "file_count": 111 + "estimated_tokens": 393459, + "file_count": 113 } ], "deferred_resource_governance": { - "status": "governed", + "status": "needs-review", "large_dir_count": 1, - "governed_large_dir_count": 1, + "governed_large_dir_count": 0, "directories": [ { - "status": "governed", + "status": "needs-review", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ - "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [], + "missing": [ + "trust report covers scripts" + ], "path": "scripts", - "estimated_tokens": 387626, - "file_count": 111, + "estimated_tokens": 393459, + "file_count": 113, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "Large deferred resources are indexed and backed by evidence." + "summary": "One or more large deferred resource directories still need explicit governance evidence." }, "quality_density": 135.4, "unused_resource_dirs": [], "ok": true, - "warnings": [] + "warnings": [ + "Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." + ] }, { "label": "complex-release-orchestrator", diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 467ffb09..e35aeac5 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 619, + "archive_entry_count": 623, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 1d448c2e..2ba7e47f 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.26, + "duration_ms": 26.42, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.84, + "duration_ms": 26.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.71, + "duration_ms": 29.52, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.07, + "duration_ms": 27.97, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.39, + "duration_ms": 26.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.35, + "duration_ms": 26.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.48, + "duration_ms": 26.05, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.29, + "duration_ms": 25.87, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 8db41a56..7575a4bc 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -24,15 +24,15 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | | skill-package-contract | baseline | command | local-output-eval-runner | 26.89 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.26 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.84 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.71 | 80 | 100.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.42 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 26.21 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.52 | 80 | 100.0 | pass | | ir-before-packaging | baseline | command | local-output-eval-runner | 29.67 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.07 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.39 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.35 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.48 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.29 | 98 | 100.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.97 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.12 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.11 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.05 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.87 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 578eb4f3..7c171391 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf", - "archive_entry_count": 619, + "archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e", + "archive_entry_count": 623, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index c3cd2524..629c53ae 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf` +- Archive SHA256: `5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index b7a6d032..e87898f7 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,11 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-16", "root": ".", "summary": { "target_python": "3.11", - "file_count": 177, + "file_count": 180, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -118,6 +118,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/evidence_consistency_release.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/evidence_consistency_world_class.py", "ok": true, @@ -442,6 +448,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/render_world_class_submission_review.py", "ok": true, @@ -1072,6 +1084,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_world_class_submission_review.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 362aca7d..e23add28 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,12 +1,12 @@ # Python Compatibility -Generated at: `2026-06-13` +Generated at: `2026-06-16` ## Summary - decision: `pass` - target python: `3.11` -- files scanned: `177` +- files scanned: `180` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index a0768d39..a63ee43c 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544", - "archive_sha256": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf" + "package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1", + "archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf", + "archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544" + "package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index ab3c4260..82d777cd 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544` -- Archive SHA256: `77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf` +- Package SHA256: `909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1` +- Archive SHA256: `5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 74871c9c..5de5c58e 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 199, - "script_count": 112, + "scanned_files": 200, + "script_count": 113, "internal_module_count": 30, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 69, + "file_write_script_count": 70, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 82, + "help_smoke_checked_count": 83, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 199, - "package_sha256": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544" + "package_hash_file_count": 200, + "package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1" }, "failures": [], "warnings": [], @@ -998,6 +998,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_preflight.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_world_class_submission_review.py", "interface": "cli", @@ -1645,9 +1659,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 82, - "checked_count": 82, - "passed_count": 82, + "candidate_count": 83, + "checked_count": 83, + "passed_count": 83, "failed_count": 0, "skipped_count": 30, "failed_scripts": [], @@ -2302,6 +2316,16 @@ "stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_preflight.py", + "command": "python3 scripts/render_world_class_preflight.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_preflight.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n [--output-md OU", + "stderr_excerpt": "" + }, { "path": "scripts/render_world_class_submission_review.py", "command": "python3 scripts/render_world_class_submission_review.py --help", @@ -2705,6 +2729,7 @@ "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 6ca6a940..f63679b3 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `199` -- Scripts: `112` +- Scanned files: `200` +- Scripts: `113` - Internal script modules: `30` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `69` +- File-write scripts: `70` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `82` +- CLI help smoke checked: `83` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `199` -- Package SHA256: `ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544` +- Package hash files: `200` +- Package SHA256: `909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `82` -- Passed scripts: `82` +- Checked scripts: `83` +- Passed scripts: `83` - Failed scripts: `none` ## Script Surface @@ -130,6 +130,7 @@ | scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. | | scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_world_class_operator_runbook.py | cli | True | True | True | False | False | True | False | Renders an operator runbook for collecting pending world-class evidence without accepting evidence. | +| scripts/render_world_class_preflight.py | cli | True | True | True | False | False | True | False | Renders a preflight checklist for collecting pending world-class evidence without accepting evidence. | | scripts/render_world_class_submission_review.py | cli | True | True | True | False | False | True | False | Renders a read-only review queue for world-class evidence submissions before ledger acceptance. | | scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/review_studio_action_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 791d9d0a..64aa9864 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -114,26 +114,26 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `30` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `82 / 82` CLI help smoke checks passing across `112` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `30` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `83 / 83` CLI help smoke checks passing across `113` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `619` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `619` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `623` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `623` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0, and world-class evidence gaps must be closed with accepted evidence rather than waived as completed | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | -| Review Studio | decision `review`, world-class score `91`, `16` gates, `0` blockers, `3` warnings, `3` review actions, `0` open annotation blockers | +| Review Studio | decision `review`, world-class score `89`, `16` gates, `0` blockers, `4` warnings, `3` review actions, `0` open annotation blockers | | Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | | World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | | World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion | | Benchmark Reproducibility | local reproducibility ready with `24` required artifacts, `0` missing artifacts, `22` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations | | IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | | Context Budget | initial load `960/1000`, under the production budget | -| CI | `make ci-test` target count is `76` after the roadmap count alignment gate update | +| CI | `make ci-test` target count is `77` after the world-class preflight gate update | ## Next Highest-Leverage Moves diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index c5bb9d6c..16548207 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "112 scripts; secrets 0; help failures 0", + "current": "113 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -204,7 +204,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 619; install failures 0", + "current": "archive entries 623; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -613,7 +613,7 @@ "label": "Evidence Consistency", "status": "pass", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "29 consistency checks", + "current": "31 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 78cdd70e..491b5718 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,9 +26,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 112 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 113 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 619; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 623; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | @@ -48,7 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | | Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | | Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | -| Evidence Consistency | `pass` | 29 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | +| Evidence Consistency | `pass` | 31 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | ## Reference Extension Tracks diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 83d174c1..b8132d54 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "77d9bd77e4a9e3992b727da369e195d94753fd4195b33981abadd10e81ee9fdf" + "to": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "ce3d8c67706a5059ae162d937f51d7469aee3d0095a53889c686524f2b43d544" + "to": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1" } ] }, diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index a541ef6c..1d74c9be 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -6,9 +6,9 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 173, - "json_claim_surface_count": 85, - "metadata_claim_surface_count": 86, + "claim_surface_count": 175, + "json_claim_surface_count": 86, + "metadata_claim_surface_count": 87, "package_claim_surface_count": 17, "violation_count": 0, "overclaim_guard_active": true, @@ -670,6 +670,14 @@ "path": "reports/world_class_evidence_plan.md", "violation_count": 0 }, + { + "path": "reports/world_class_evidence_preflight.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.md", + "violation_count": 0 + }, { "path": "reports/world_class_operator_runbook.html", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index b7d0d554..4fbacd98 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -7,9 +7,9 @@ Generated at: `2026-06-16` - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `173` -- JSON claim surfaces scanned: `85` -- metadata claim surfaces scanned: `86` +- claim surfaces scanned: `175` +- JSON claim surfaces scanned: `86` +- metadata claim surfaces scanned: `87` - package/runtime claim surfaces scanned: `17` - violations: `0` - overclaim guard active: `true` diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json new file mode 100644 index 00000000..1bdf5533 --- /dev/null +++ b/reports/world_class_evidence_preflight.json @@ -0,0 +1,733 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-16", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "precheck_count": 13, + "precheck_pass_count": 8, + "precheck_missing_count": 1, + "precheck_optional_count": 1, + "precheck_human_required_count": 1, + "precheck_external_required_count": 2, + "collection_ready_count": 1, + "collection_blocked_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "pending_count": 4, + "ready_to_claim_world_class": false, + "credential_value_exposed": false, + "preflight_counts_as_evidence": false, + "decision": "collection-preflight-blocked" + }, + "status_counts": { + "blocked": 3, + "ready-for-human-review": 1 + }, + "precheck_status_counts": { + "pass": 8, + "missing": 1, + "optional": 1, + "human-required": 1, + "external-required": 2 + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "openai-api-key", + "label": "Provider credential", + "kind": "env", + "required": true, + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "secret_value_redacted": true, + "env": "OPENAI_API_KEY", + "status": "missing", + "actual": "not-set", + "default": "" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "gpt-4.1-mini" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "ready-for-human-review", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Fill winner_variant values with real A/B decisions.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + } + ], + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + } + ], + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "openai-api-key", + "label": "Provider credential", + "kind": "env", + "required": true, + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "secret_value_redacted": true, + "env": "OPENAI_API_KEY", + "status": "missing", + "actual": "not-set", + "default": "" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "gpt-4.1-mini" + }, + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Fill winner_variant values with real A/B decisions.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + }, + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + }, + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + }, + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "submissions": { + "directory": "evidence/world_class/submissions", + "preflight_counts_submission_as_completion": false + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json" + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md" + } +} diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md new file mode 100644 index 00000000..18ee86ae --- /dev/null +++ b/reports/world_class_evidence_preflight.md @@ -0,0 +1,120 @@ +# World-Class Evidence Preflight + +Generated at: `2026-06-16` + +## Summary + +- decision: `collection-preflight-blocked` +- ready to claim world-class: `false` +- preflight counts as evidence: `false` +- credential value exposed: `false` +- collection ready: `1` +- collection blocked: `3` +- source checks: `6` pass / `13` total + +This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger. + +## Evidence Items + +| Evidence | Status | Intake | Review | Next action | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `blocked` | `awaiting-submission` | `awaiting-submission` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | +| `human-adjudication` | `ready-for-human-review` | `awaiting-submission` | `awaiting-submission` | Assign a real reviewer identity before claiming human adjudication. | +| `native-permission-enforcement` | `blocked` | `awaiting-submission` | `awaiting-submission` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | +| `native-client-telemetry` | `blocked` | `awaiting-submission` | `awaiting-submission` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | + +## Provider Holdout + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/provider-holdout.json` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Output eval cases | `file` | `present` | `pass` | Keep output holdout cases available before provider execution. | +| Provider runner | `file` | `present` | `pass` | Use the provider runner instead of the local command runner for model-backed evidence. | +| Provider credential | `env` | `not-set` | `missing` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | +| Provider model | `env` | `not-set` | `optional` | Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Provider model run | `0` | `>0` | `blocked` | Run provider-backed output-exec with real credentials. | +| Timing observed | `10` | `>0` | `pass` | Provider execution should record timing metadata. | +| Token usage observed | `0` | `>0` | `blocked` | Provider execution should return non-estimated token usage. | + +## Human Adjudication + +- status: `ready-for-human-review` +- ledger: `pending` +- submission: `evidence/world_class/submissions/human-adjudication.json` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Blind review kit | `file` | `present` | `pass` | Open the blind review kit and record real reviewer choices. | +| Decision template | `file` | `present` | `pass` | Fill winner_variant values with real A/B decisions. | +| Human reviewer | `human` | `external-human-action` | `human-required` | Assign a real reviewer identity before claiming human adjudication. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Review pairs exist | `5` | `>0` | `pass` | Generate the blind A/B review pack. | +| No pending decisions | `5` | `==0` | `blocked` | Record a reviewer choice for every pair. | +| Judgments complete | `0` | `==pair_count` | `blocked` | Every pair needs one valid human judgment. | +| No invalid decisions | `0` | `==0` | `pass` | Fix malformed winner/confidence entries. | + +## Native Permission Enforcement + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/native-permission-enforcement.json` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Permission policy | `file` | `present` | `pass` | Keep approved high-permission capabilities explicit. | +| Runtime probes | `file` | `present` | `pass` | Refresh runtime permission probes after packaging changes. | +| Native guard | `external` | `external-integration-required` | `external-required` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Native enforcement | `0` | `>0` | `blocked` | Collect real target-client or external runtime guard proof. | +| Probe failures | `0` | `==0` | `pass` | Runtime permission probes must stay clean. | +| Installer support | `True` | `true` | `pass` | Installer enforcement is supporting evidence, not native proof. | + +## Native Client Telemetry + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/native-client-telemetry.json` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Native telemetry host | `file` | `present` | `pass` | Use the native host to receive metadata-only client events. | +| Hook recipes | `file` | `present` | `pass` | Refresh telemetry hook recipes before external client installation. | +| External client | `external` | `external-integration-required` | `external-required` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | +| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | + +## Boundary + +- Environment variables are reported only as `set` or `not-set`; values are never printed. +- Human-required and external-required states are operator actions, not accepted evidence. +- The world-class ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 254605f2..7a34f501 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -59,6 +59,7 @@ DEFAULT_TARGETS = [ "world-class-evidence-check", "world-class-ledger-check", "world-class-intake-check", + "world-class-preflight-check", "world-class-submission-review-check", "world-class-runbook-check", "world-class-claim-guard-check", diff --git a/scripts/evidence_consistency_release.py b/scripts/evidence_consistency_release.py index e7461812..7a8145da 100644 --- a/scripts/evidence_consistency_release.py +++ b/scripts/evidence_consistency_release.py @@ -13,6 +13,7 @@ FIRST_CLASS_REPORT_COMMANDS = [ "python3 scripts/render_skill_overview.py .", "python3 scripts/render_skill_interpretation.py .", "python3 scripts/render_review_viewer.py .", + 'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"', "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json", 'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"', ] @@ -62,6 +63,7 @@ def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]: "reports/skill-overview.json", "reports/skill-interpretation.json", "reports/review-viewer.json", + "reports/world_class_evidence_preflight.json", "reports/review-studio.json", "reports/evidence_consistency.json", ], diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 59d1bfcc..05579046 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -21,6 +21,7 @@ REQUIRED_REPORTS = { "world_class_ledger": "reports/world_class_evidence_ledger.json", "world_class_plan": "reports/world_class_evidence_plan.json", "world_class_intake": "reports/world_class_evidence_intake.json", + "world_class_preflight": "reports/world_class_evidence_preflight.json", "world_class_submission_review": "reports/world_class_submission_review.json", "world_class_operator_runbook": "reports/world_class_operator_runbook.json", "skill_os2_coverage": "reports/skill_os2_coverage.json", @@ -260,6 +261,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: ledger = reports["world_class_ledger"] world_class_plan = reports["world_class_plan"] world_class_intake = reports["world_class_intake"] + world_class_preflight = reports["world_class_preflight"] world_class_submission_review = reports["world_class_submission_review"] world_class_operator_runbook = reports["world_class_operator_runbook"] coverage = reports["skill_os2_coverage"] @@ -280,6 +282,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: trust_summary = nested(trust, ["summary"], {}) context_stats = nested(context_budget, ["stats"], {}) claim_guard_summary = nested(claim_guard, ["summary"], {}) + preflight_summary = nested(world_class_preflight, ["summary"], {}) if isinstance(benchmark_summary, dict): compare_values( checks, @@ -430,6 +433,28 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], detail="Benchmark reproducibility must not overstate public claim readiness.", ) + preflight_boundary = { + "pending_count": ledger_summary.get("pending_count"), + "source_check_count": ledger_summary.get("source_check_count"), + "source_pass_count": ledger_summary.get("source_pass_count"), + "source_blocked_count": ledger_summary.get("source_blocked_count"), + "ready_to_claim_world_class": ledger_summary.get("ready_to_claim_world_class"), + "preflight_counts_as_evidence": False, + "credential_value_exposed": False, + } + actual_preflight_boundary = { + key: preflight_summary.get(key) if isinstance(preflight_summary, dict) else None + for key in preflight_boundary + } + compare_values( + checks, + key="preflight-world-class-boundary", + label="Preflight mirrors ledger without accepting evidence", + expected=preflight_boundary, + actual=actual_preflight_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]], + detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.", + ) public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False compare_values( diff --git a/scripts/render_world_class_preflight.py b/scripts/render_world_class_preflight.py new file mode 100644 index 00000000..807852af --- /dev/null +++ b/scripts/render_world_class_preflight.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger +from render_world_class_submission_review import build_submission_review +from world_class_evidence_contract import rel_path +from world_class_source_checks import summarize_source_checklist + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence." + + +PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = { + "provider-holdout": [ + { + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "path": "evals/output/cases.jsonl", + "required": True, + "next_action": "Keep output holdout cases available before provider execution.", + }, + { + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "path": "scripts/provider_output_eval_runner.py", + "required": True, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + }, + { + "key": "openai-api-key", + "label": "Provider credential", + "kind": "env", + "name": "OPENAI_API_KEY", + "required": True, + "secret": True, + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + }, + { + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "name": "YAO_OUTPUT_EVAL_MODEL", + "required": False, + "default": "gpt-4.1-mini", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + }, + ], + "human-adjudication": [ + { + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "path": "reports/output_review_kit.html", + "required": True, + "next_action": "Open the blind review kit and record real reviewer choices.", + }, + { + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "path": "reports/output_review_decisions.json", + "required": True, + "next_action": "Fill winner_variant values with real A/B decisions.", + }, + { + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": True, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + }, + ], + "native-permission-enforcement": [ + { + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "path": "security/permission_policy.json", + "required": True, + "next_action": "Keep approved high-permission capabilities explicit.", + }, + { + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "path": "reports/runtime_permission_probes.json", + "required": True, + "next_action": "Refresh runtime permission probes after packaging changes.", + }, + { + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": True, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + }, + ], + "native-client-telemetry": [ + { + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "path": "scripts/telemetry_native_host.py", + "required": True, + "next_action": "Use the native host to receive metadata-only client events.", + }, + { + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "path": "reports/telemetry_hook_recipes.json", + "required": True, + "next_action": "Refresh telemetry hook recipes before external client installation.", + }, + { + "key": "external-client", + "label": "External client", + "kind": "external", + "required": True, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + }, + ], +} + + +def env_present(name: str) -> bool: + return bool(os.environ.get(name)) + + +def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) -> dict[str, Any]: + kind = str(spec.get("kind", "")) + required = spec.get("required") is True + row = { + "evidence_key": evidence_key, + "key": spec.get("key", ""), + "label": spec.get("label", ""), + "kind": kind, + "required": required, + "next_action": spec.get("next_action", ""), + "secret_value_redacted": spec.get("secret") is True, + } + if kind == "file": + path = skill_dir / str(spec.get("path", "")) + exists = path.exists() + row.update( + { + "path": rel_path(path, skill_dir), + "status": "pass" if exists else ("missing" if required else "optional"), + "actual": "present" if exists else "missing", + } + ) + return row + if kind == "env": + name = str(spec.get("name", "")) + present = env_present(name) + row.update( + { + "env": name, + "status": "pass" if present else ("missing" if required else "optional"), + "actual": "set" if present else "not-set", + "default": spec.get("default", ""), + } + ) + return row + if kind == "human": + row.update( + { + "status": "human-required", + "actual": "external-human-action", + } + ) + return row + if kind == "external": + row.update( + { + "status": "external-required", + "actual": "external-integration-required", + } + ) + return row + row.update({"status": "unknown", "actual": "unknown"}) + return row + + +def item_status(entry: dict[str, Any], prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str: + if entry.get("status") == "accepted": + return "accepted" + blocking = {row.get("status") for row in prechecks if row.get("required") is True} + if "missing" in blocking or "external-required" in blocking: + return "blocked" + if "human-required" in blocking: + return "ready-for-human-review" + if any(row.get("status") != "pass" for row in source_rows): + return "ready-to-collect" + return "ready-for-submission" + + +def first_next_action(prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str: + for row in prechecks: + if row.get("required") is True and row.get("status") != "pass": + return str(row.get("next_action", "")) + for row in source_rows: + if row.get("status") != "pass": + return str(row.get("next_action", "")) + return "Validate intake, refresh the ledger, and run the claim guard." + + +def md_cell(value: Any) -> str: + return str(value).replace("|", "\\|") + + +def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir) + review_by_key = {str(item.get("evidence_key", "")): item for item in review.get("items", [])} + intake_by_key = {str(item.get("evidence_key", "")): item for item in intake.get("operator_checklist", [])} + items: list[dict[str, Any]] = [] + precheck_rows: list[dict[str, Any]] = [] + source_rows: list[dict[str, Any]] = [] + for entry in ledger.get("entries", []): + key = str(entry.get("key", "")) + prechecks = [build_precheck(skill_dir, key, spec) for spec in PREFLIGHT_SPECS.get(key, [])] + review_item = review_by_key.get(key, {}) + item_source_rows = review_item.get("source_checklist", entry.get("source_checklist", [])) + item_source_rows = item_source_rows if isinstance(item_source_rows, list) else [] + status = item_status(entry, prechecks, item_source_rows) + item = { + "evidence_key": key, + "label": entry.get("label", key), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "status": status, + "ledger_status": entry.get("status", "pending"), + "intake_readiness": intake_by_key.get(key, {}).get("readiness", "missing"), + "review_state": review_item.get("review_state", "missing"), + "source_accepted": entry.get("source_accepted") is True, + "prechecks": prechecks, + "source_checklist": item_source_rows, + "next_action": first_next_action(prechecks, item_source_rows), + "submission_path": intake_by_key.get(key, {}).get("submission_path", ""), + "template_path": intake_by_key.get(key, {}).get("template_path", ""), + "runbook": entry.get("runbook", []), + } + items.append(item) + precheck_rows.extend(prechecks) + source_rows.extend(item_source_rows) + precheck_status_counts: dict[str, int] = {} + for row in precheck_rows: + status = str(row.get("status", "unknown")) + precheck_status_counts[status] = precheck_status_counts.get(status, 0) + 1 + source_summary = summarize_source_checklist(source_rows) + collection_ready_count = sum(1 for item in items if item["status"] in {"ready-to-collect", "ready-for-human-review", "ready-for-submission"}) + blocked_count = sum(1 for item in items if item["status"] == "blocked") + ready_to_claim = ledger.get("summary", {}).get("ready_to_claim_world_class") is True + summary = { + "evidence_item_count": len(items), + "precheck_count": len(precheck_rows), + "precheck_pass_count": precheck_status_counts.get("pass", 0), + "precheck_missing_count": precheck_status_counts.get("missing", 0), + "precheck_optional_count": precheck_status_counts.get("optional", 0), + "precheck_human_required_count": precheck_status_counts.get("human-required", 0), + "precheck_external_required_count": precheck_status_counts.get("external-required", 0), + "collection_ready_count": collection_ready_count, + "collection_blocked_count": blocked_count, + **source_summary, + "pending_count": ledger.get("summary", {}).get("pending_count", 0), + "ready_to_claim_world_class": ready_to_claim, + "credential_value_exposed": False, + "preflight_counts_as_evidence": False, + "decision": "ready-for-completion-audit" if ready_to_claim else ("collection-preflight-blocked" if blocked_count else "ready-to-collect-evidence"), + } + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": {item["status"]: sum(1 for candidate in items if candidate["status"] == item["status"]) for item in items}, + "precheck_status_counts": precheck_status_counts, + "items": items, + "prechecks": precheck_rows, + "source_checklist": source_rows, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "preflight_counts_submission_as_completion": False, + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json", + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Evidence Preflight", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- preflight counts as evidence: `{str(summary['preflight_counts_as_evidence']).lower()}`", + f"- credential value exposed: `{str(summary['credential_value_exposed']).lower()}`", + f"- collection ready: `{summary['collection_ready_count']}`", + f"- collection blocked: `{summary['collection_blocked_count']}`", + f"- source checks: `{summary['source_pass_count']}` pass / `{summary['source_check_count']}` total", + "", + "This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.", + "", + "## Evidence Items", + "", + "| Evidence | Status | Intake | Review | Next action |", + "| --- | --- | --- | --- | --- |", + ] + for item in report["items"]: + next_action = str(item.get("next_action", "")).replace("|", "\\|") + lines.append( + f"| `{item['evidence_key']}` | `{item['status']}` | `{item['intake_readiness']}` | `{item['review_state']}` | {next_action} |" + ) + for item in report["items"]: + lines.extend( + [ + "", + f"## {item['label']}", + "", + f"- status: `{item['status']}`", + f"- ledger: `{item['ledger_status']}`", + f"- submission: `{item.get('submission_path') or 'missing'}`", + "", + "### Prechecks", + "", + "| Check | Kind | Current | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in item.get("prechecks", []): + action = md_cell(row.get("next_action", "")) + lines.append( + f"| {row['label']} | `{row['kind']}` | `{row['actual']}` | `{row['status']}` | {action} |" + ) + lines.extend( + [ + "", + "### Source Checks", + "", + "| Check | Current | Expected | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in item.get("source_checklist", []): + action = md_cell(row.get("next_action", "")) + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` | {action} |" + ) + lines.extend( + [ + "", + "## Boundary", + "", + "- Environment variables are reported only as `set` or `not-set`; values are never printed.", + "- Human-required and external-required states are operator actions, not accepted evidence.", + "- The world-class ledger remains the source of truth for `ready_to_claim_world_class`.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render collection preflight checks for pending world-class evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_evidence_preflight.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_preflight.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_preflight(skill_dir, args.generated_at, submissions_dir=submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/yao.py b/scripts/yao.py index 50a2538d..26b28fac 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -56,6 +56,7 @@ from yao_cli_report_commands import ( command_world_class_evidence, command_world_class_intake, command_world_class_ledger, + command_world_class_preflight, command_world_class_runbook, command_world_class_submission_kit, command_world_class_submission_review, diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 64709ffb..8bd6b3be 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -217,6 +217,17 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] world_class_intake_cmd.add_argument("--generated-at") world_class_intake_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_intake")) + world_class_preflight_cmd = subparsers.add_parser( + "world-class-preflight", + help="Render operator preflight checks for collecting pending world-class evidence.", + ) + world_class_preflight_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_preflight_cmd.add_argument("--submissions-dir") + world_class_preflight_cmd.add_argument("--output-json") + world_class_preflight_cmd.add_argument("--output-md") + world_class_preflight_cmd.add_argument("--generated-at") + world_class_preflight_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_preflight")) + world_class_submission_kit_cmd = subparsers.add_parser( "world-class-submission-kit", help="Prepare editable world-class evidence submission drafts.", diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index 8706c8ea..79710f43 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -72,6 +72,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_world_class_evidence_plan.py", [str(ROOT)]), run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), run_script("render_world_class_evidence_intake.py", [str(ROOT)]), + run_script("render_world_class_preflight.py", [str(ROOT)]), run_script("render_world_class_submission_review.py", [str(ROOT)]), run_script("render_world_class_operator_runbook.py", [str(ROOT)]), run_script("render_world_class_claim_guard.py", [str(ROOT)]), @@ -114,6 +115,7 @@ def command_report(args: argparse.Namespace) -> int: "world_class_evidence_plan": "reports/world_class_evidence_plan.json", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", "world_class_evidence_intake": "reports/world_class_evidence_intake.json", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.json", "world_class_submission_review": "reports/world_class_submission_review.json", "world_class_operator_runbook": "reports/world_class_operator_runbook.json", "world_class_claim_guard": "reports/world_class_claim_guard.json", @@ -175,6 +177,14 @@ def command_world_class_intake(args: argparse.Namespace) -> int: return emit_result(run_script("render_world_class_evidence_intake.py", cmd)) +def command_world_class_preflight(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_preflight.py", cmd)) + + def command_world_class_submission_kit(args: argparse.Namespace) -> int: cmd = [resolved_skill_dir(args)] if args.output_dir: diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 7835338f..7c45da74 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -20,6 +20,7 @@ REPORT_FILES = [ "reports/world_class_evidence_ledger.json", "reports/world_class_evidence_plan.json", "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_preflight.json", "reports/world_class_submission_review.json", "reports/world_class_operator_runbook.json", "reports/skill_os2_coverage.json", @@ -53,6 +54,7 @@ def refresh_embedded_reports() -> None: "render_world_class_evidence_plan.py", "render_world_class_evidence_ledger.py", "render_world_class_evidence_intake.py", + "render_world_class_preflight.py", "render_world_class_submission_review.py", "render_world_class_operator_runbook.py", "render_world_class_claim_guard.py", @@ -107,6 +109,7 @@ def assert_release_evidence_instructions_cover_first_class_reports() -> None: clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0] for block in [source_refresh, clean_lock]: assert "python3 scripts/render_skill_interpretation.py ." in block, block + assert "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"" in block, block assert "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"" in block, block @@ -137,7 +140,7 @@ def main() -> None: assert payload["ok"] is True, payload assert payload["summary"]["decision"] == "consistent", payload assert payload["summary"]["fail_count"] == 0, payload - assert payload["summary"]["check_count"] >= 30, payload + assert payload["summary"]["check_count"] >= 31, payload checks = {item["key"]: item for item in payload["checks"]} assert checks["release-evidence-flow-covers-first-class-reports"]["status"] == "pass", checks[ "release-evidence-flow-covers-first-class-reports" @@ -145,6 +148,7 @@ def main() -> None: assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] + assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"] assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"] assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[ "claim-guard-package-runtime-surface" @@ -216,6 +220,32 @@ def main() -> None: claim_guard_drift_checks["claim-guard-package-runtime-surface"] ) + preflight_drift_root = TMP / "preflight-drift-skill" + copy_reports(preflight_drift_root) + preflight_path = preflight_drift_root / "reports" / "world_class_evidence_preflight.json" + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + preflight["summary"]["credential_value_exposed"] = True + preflight_path.write_text(json.dumps(preflight, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + preflight_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(preflight_drift_root), + "--output-json", + str(TMP / "preflight_drift.json"), + "--output-md", + str(TMP / "preflight_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert preflight_drift_proc.returncode == 2, preflight_drift_proc.stdout + preflight_drift_payload = json.loads(preflight_drift_proc.stdout) + preflight_drift_checks = {item["key"]: item for item in preflight_drift_payload["checks"]} + assert preflight_drift_checks["preflight-world-class-boundary"]["status"] == "fail", ( + preflight_drift_checks["preflight-world-class-boundary"] + ) + release_flow_drift_root = TMP / "release-flow-drift-skill" copy_reports(release_flow_drift_root) agents_path = release_flow_drift_root / "AGENTS.md" diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 7102136d..871265de 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -103,6 +103,7 @@ def provider_submission(*, valid: bool, artifact_path: str = "reports/output_exe def assert_documented_submission_commands() -> None: expected_fragments = [ 'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"', + 'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"', 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"', 'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"', 'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"', diff --git a/tests/verify_world_class_preflight.py b/tests/verify_world_class_preflight.py new file mode 100644 index 00000000..287e864a --- /dev/null +++ b/tests/verify_world_class_preflight.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_preflight.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_world_class_preflight" + + +def run_preflight(extra_env: dict[str, str] | None = None, *extra: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ) + env.pop("OPENAI_API_KEY", None) + env.pop("YAO_OUTPUT_EVAL_MODEL", None) + if extra_env: + env.update(extra_env) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-16", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + + +def run_cli(*extra: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("OPENAI_API_KEY", None) + return subprocess.run( + [sys.executable, str(CLI), "world-class-preflight", str(ROOT), *extra], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + + +def by_key(items: list[dict], key: str) -> dict: + return next(item for item in items if item.get("evidence_key") == key) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + + output_json = TMP / "world_class_evidence_preflight.json" + output_md = TMP / "world_class_evidence_preflight.md" + proc = run_preflight(None, "--output-json", str(output_json), "--output-md", str(output_md)) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "collection-preflight-blocked", summary + assert summary["ready_to_claim_world_class"] is False, summary + assert summary["preflight_counts_as_evidence"] is False, summary + assert summary["credential_value_exposed"] is False, summary + assert summary["evidence_item_count"] == 4, summary + assert summary["pending_count"] == 4, summary + assert summary["precheck_count"] >= 12, summary + assert summary["precheck_missing_count"] >= 1, summary + assert summary["precheck_external_required_count"] == 2, summary + assert summary["precheck_human_required_count"] == 1, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary + assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload + + provider = by_key(payload["items"], "provider-holdout") + assert provider["status"] == "blocked", provider + provider_checks = {item["key"]: item for item in provider["prechecks"]} + assert provider_checks["openai-api-key"]["status"] == "missing", provider_checks + assert provider_checks["openai-api-key"]["actual"] == "not-set", provider_checks + assert provider_checks["openai-api-key"]["secret_value_redacted"] is True, provider_checks + assert "sk-test-secret" not in proc.stdout, proc.stdout + assert "OPENAI_API_KEY" in proc.stdout, proc.stdout + assert "set OPENAI_API_KEY" not in proc.stdout.lower(), proc.stdout + + human = by_key(payload["items"], "human-adjudication") + assert human["status"] == "ready-for-human-review", human + human_checks = {item["key"]: item for item in human["prechecks"]} + assert human_checks["human-reviewer"]["status"] == "human-required", human_checks + assert "reviewer identity" in human["next_action"], human + assert any("Record a reviewer choice" in row["next_action"] for row in human["source_checklist"]), human + + native = by_key(payload["items"], "native-permission-enforcement") + assert native["status"] == "blocked", native + assert any(item["status"] == "external-required" for item in native["prechecks"]), native + + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Evidence Preflight" in markdown, markdown + assert "ready to claim world-class: `false`" in markdown, markdown + assert "preflight counts as evidence: `false`" in markdown, markdown + assert "credential value exposed: `false`" in markdown, markdown + assert "values are never printed" in markdown, markdown + + env_json = TMP / "preflight_with_env.json" + env_proc = run_preflight( + {"OPENAI_API_KEY": "sk-test-secret", "YAO_OUTPUT_EVAL_MODEL": "gpt-test"}, + "--output-json", + str(env_json), + "--output-md", + str(TMP / "preflight_with_env.md"), + ) + env_payload = json.loads(env_proc.stdout) + env_provider = by_key(env_payload["items"], "provider-holdout") + env_provider_checks = {item["key"]: item for item in env_provider["prechecks"]} + assert env_provider_checks["openai-api-key"]["status"] == "pass", env_provider_checks + assert env_provider_checks["openai-api-key"]["actual"] == "set", env_provider_checks + assert env_provider_checks["provider-model"]["status"] == "pass", env_provider_checks + assert "sk-test-secret" not in env_proc.stdout, env_proc.stdout + assert env_payload["summary"]["credential_value_exposed"] is False, env_payload + assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload + + cli_proc = run_cli( + "--output-json", + str(TMP / "cli_preflight.json"), + "--output-md", + str(TMP / "cli_preflight.md"), + "--generated-at", + "2026-06-16", + ) + cli_payload = json.loads(cli_proc.stdout) + assert cli_payload["summary"]["decision"] == "collection-preflight-blocked", cli_payload + assert cli_payload["summary"]["preflight_counts_as_evidence"] is False, cli_payload + assert (TMP / "cli_preflight.md").exists(), cli_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 639b63f4..0253fb0f 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -80,7 +80,7 @@ def main() -> None: parser_help = yao_cli_module.build_parser().format_help() expected_help = ( "quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " - "world-class-evidence world-class-ledger world-class-intake world-class-submission-kit " + "world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit " "world-class-submission-review world-class-runbook world-class-claim-guard " "benchmark-reproducibility evidence-consistency output-review-kit adapt-scan adapt-propose adapt-apply " "telemetry-import telemetry-emit telemetry-hooks --record-cli-telemetry"