diff --git a/registry/index.json b/registry/index.json index 2f65c55f..3e25dffd 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index a935113d..64544c13 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index b10d2e5b..a22ccc6d 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -56,6 +56,14 @@ "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_world_class_preflight.py", + "lines": 590, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "tests/verify_review_studio.py", "lines": 590, @@ -111,14 +119,6 @@ "severity": "pass", "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, - { - "path": "scripts/render_review_studio.py", - "lines": 568, - "kind": "cli-script", - "severity": "pass", - "early_watch": false, - "recommendation": "Move data loading and large section renderers into focused review_studio_* modules." } ], "watchlist": [], diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index f69586e3..26201305 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -46,6 +46,7 @@ No near-threshold files found. | `tests/verify_world_class_evidence_intake.py` | `690` | `test` | `pass` | | `tests/verify_output_review_adjudication.py` | `599` | `test` | `pass` | | `scripts/render_benchmark_reproducibility.py` | `595` | `cli-script` | `pass` | +| `scripts/render_world_class_preflight.py` | `590` | `cli-script` | `pass` | | `tests/verify_review_studio.py` | `590` | `test` | `pass` | | `scripts/render_skill_overview.py` | `588` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `586` | `cli-script` | `pass` | @@ -53,7 +54,6 @@ No near-threshold files found. | `scripts/trust_check.py` | `582` | `cli-script` | `pass` | | `tests/verify_world_class_evidence_ledger.py` | `579` | `test` | `pass` | | `scripts/render_evidence_consistency.py` | `568` | `cli-script` | `pass` | -| `scripts/render_review_studio.py` | `568` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 30633801..5caaa4fd 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,24 +3,24 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "git_status": { "available": true, "dirty": true, - "changed_file_count": 51, + "changed_file_count": 34, "sample": [ " M registry/index.json", " M registry/packages/yao-meta-skill.json", - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/adoption_drift_report.json", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/context_budget.json", " M reports/context_budget.md", - " M reports/context_budget_summary.json" + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md" ], "scope": "generation-time status before this report is written" }, @@ -30,9 +30,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -52,7 +52,7 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, "public_claim": { "ready": false, @@ -68,7 +68,7 @@ }, "release_lock": { "ready": false, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "status_scope": "generation-time status before this report is written", "reason": "working tree was dirty at generation time" }, @@ -78,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61" + "sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -152,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7966, - "sha256": "baf406c10747ebb03a12f0bc74233c3e98cf1715bebfcdb450286ac1b2f5277f" + "sha256": "874d573922462a290776489b7ab760136522b2780339b579e383ab8235af56e8" }, { "label": "blind_review", @@ -187,7 +187,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 134467, - "sha256": "875af23304269c89191e86f8c41e30cfbf5c9bf9603ae478c03bbc163b2a13ef" + "sha256": "d2e8352cae2c29a009d81bfc78ec4478cc36f045022d0803c7b229d6441fe1e9" }, { "label": "python_compatibility", @@ -201,14 +201,14 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "8707b96ff516f0e52201e9b2fa3fbbc59fdfb3046d7d5f534eae47c780563a93" + "sha256": "d0c088f66524fa16c604b88fcd45010303daed7eb9e5273e439451484c09442e" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "d8595fc2c41a897e607eae7a843bbdc28a2941b04d1e913ac9d2dbae233b7a2a" + "sha256": "d18b6336d485f6290b68bf229d491bcd8a08ad872a8e9a53502b996c4f9c1347" }, { "label": "install_simulation", @@ -222,7 +222,7 @@ "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14466, - "sha256": "b1c82626a1611c22c4d4cf3a6caa96a80355e9ecbdaee13a683456e648702406" + "sha256": "e6f644b1d2f590e76c7f2dd5a4888efad138bd2a2643c0fa60fe2fea81db55e1" }, { "label": "world_class_evidence_plan", @@ -249,8 +249,8 @@ "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 46188, - "sha256": "0c140e75d4146092b522ec65514d055c9e1cab9f543bc81b828f5a1b6ec712b8" + "bytes": 57313, + "sha256": "585ec2f93a1dfce5b270af048e3d551676b93231477d7bf4cfe5b08946e82619" }, { "label": "world_class_submission_review", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index d38737c3..eda15d49 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,9 +1,9 @@ # Benchmark Reproducibility Generated at: `2026-06-17` -Commit: `2241b0e344e896e26a1b8264a797e45665613b8b` +Commit: `48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb` Working tree dirty at generation: `true` -Evidence bundle SHA256: `53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61` +Evidence bundle SHA256: `fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119` ## Summary @@ -12,8 +12,8 @@ Evidence bundle SHA256: `53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `6beb1e623996` -- archive sha256: `992401b48ffe` +- source contract sha256: `cc61aeda8a4a` +- archive sha256: `b09ad63e1a0c` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `23` @@ -23,7 +23,7 @@ Evidence bundle SHA256: `53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a - world-class source checks: `9` pass / `19` total; `10` blocked - public claim ready: `false` - public claim blockers: `5` -- changed files at generation: `51` +- changed files at generation: `34` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -51,7 +51,7 @@ This report proves local benchmark reproducibility only. It keeps external provi - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61` +- sha256: `fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119` ## Methodology Sections @@ -73,21 +73,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `baf406c10747` | +| output_execution | `reports/output_execution_runs.json` | present | `874d57392246` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `91fd88dd9b0f` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `875af2330426` | +| trust_report | `reports/security_trust_report.json` | present | `d2e8352cae2c` | | python_compatibility | `reports/python_compatibility.json` | present | `0eeedc1c3ff1` | -| registry_audit | `reports/registry_audit.json` | present | `8707b96ff516` | -| package_verification | `reports/package_verification.json` | present | `d8595fc2c41a` | +| registry_audit | `reports/registry_audit.json` | present | `d0c088f66524` | +| package_verification | `reports/package_verification.json` | present | `d18b6336d485` | | install_simulation | `reports/install_simulation.json` | present | `6d808df45317` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `b1c82626a161` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `e6f644b1d2f5` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `2beb888d7992` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `22c85613d7be` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `d3f44570e605` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `0c140e75d414` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `585ec2f93a1d` | | world_class_submission_review | `reports/world_class_submission_review.json` | present | `96624a5b7f79` | | world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `a7b7809d3b42` | | world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `d72777b421b5` | diff --git a/reports/context_budget.json b/reports/context_budget.json index 35b74c5a..f780fba1 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1105271, + "other_text_tokens": 1107027, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1106068, - "deferred_resource_tokens": 503641, + "estimated_total_text_tokens": 1107824, + "deferred_resource_tokens": 504879, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index 99c82eec..35d76e6d 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 503641 | `governed` | scripts:442611 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 504879 | `governed` | scripts:443849 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 01993340..2a1b1ced 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-31", + "generated_at": "2026-06-17", "targets": [ { "label": "root", @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 503641, + "deferred_resource_tokens": 504879, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 87e3c4a0..8b7025a9 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 503641/120000; top deferred scripts 442611; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 504879/120000; top deferred scripts 443849; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 503641/120000; top deferred scripts 442611; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 504879/120000; top deferred scripts 443849; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -222,7 +222,7 @@ "actual": { "available": true, "clean": false, - "changed_file_count": 52 + "changed_file_count": 34 }, "paths": [ "reports/benchmark_reproducibility.json" @@ -263,8 +263,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "2241b0e344e896e26a1b8264a797e45665613b8b", - "actual": "2241b0e344e896e26a1b8264a797e45665613b8b", + "expected": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", + "actual": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -279,8 +279,8 @@ "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, "world_class_source_pass_count": 9, @@ -292,8 +292,8 @@ "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, "world_class_source_pass_count": 9, @@ -409,8 +409,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "2241b0e344e896e26a1b8264a797e45665613b8b", - "actual": "2241b0e344e896e26a1b8264a797e45665613b8b", + "expected": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", + "actual": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -425,8 +425,8 @@ "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, "world_class_source_pass_count": 9, @@ -438,8 +438,8 @@ "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, "world_class_source_pass_count": 9, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index d14a8f0e..2925e379 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.64, + "duration_ms": 26.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.53, + "duration_ms": 25.46, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.6, + "duration_ms": 25.94, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.48, + "duration_ms": 27.74, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.04, + "duration_ms": 28.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.52, + "duration_ms": 25.82, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.73, + "duration_ms": 26.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.71, + "duration_ms": 25.81, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.02, + "duration_ms": 25.75, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.32, + "duration_ms": 25.57, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index e6e8f3ab..a65f6016 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 26.64 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 25.53 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 25.6 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 27.48 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.04 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 25.52 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.73 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 35.71 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 29.02 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.32 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.73 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 25.46 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 25.94 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 27.74 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.3 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 25.82 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.02 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.81 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.75 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.57 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index cbf8bbef..3e86e356 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index 2ef6d62c..a8d51a70 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365` +- Archive SHA256: `b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 678ff2fa..24813bb4 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index a1f1065a..da1d449b 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e` -- Archive SHA256: `992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365` +- Package SHA256: `cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242` +- Archive SHA256: `b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index c514a265..af43196f 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -745,7 +745,7 @@

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 503641/120000; top deferred scripts 442611; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 148 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 226 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

223 Python files; 0 hotspots; 0 watchlist files; 2 early watch files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 504879/120000; top deferred scripts 443849; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 148 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 226 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

223 Python files; 0 hotspots; 0 watchlist files; 2 early watch files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -812,12 +812,12 @@
-

上下文

initial load 990/1000; deferred 503641/120000; top deferred scripts 442611; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 504879/120000; top deferred scripts 443849; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
148
网络脚本
3
Help 失败
0
包体哈希
6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e
+

信任报告

Secret
0
脚本数
148
网络脚本
3
Help 失败
0
包体哈希
cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -903,12 +903,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
686
失败数
0
警告数
0
归档哈希
992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
686
失败数
0
警告数
0
归档哈希
b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362
diff --git a/reports/review-studio.json b/reports/review-studio.json index 934e07dd..eb0dfd93 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -44,7 +44,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 503641/120000; top deferred scripts 442611; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 504879/120000; top deferred scripts 443849; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -2011,9 +2011,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -2033,9 +2033,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -2140,7 +2140,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "skill_atlas": { "skill_count": 12, @@ -2178,8 +2178,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -2210,7 +2210,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -2226,7 +2226,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 @@ -2305,12 +2305,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, @@ -4824,7 +4824,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.64, + "duration_ms": 26.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4852,7 +4852,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.53, + "duration_ms": 25.46, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4875,7 +4875,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.6, + "duration_ms": 25.94, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4903,7 +4903,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.48, + "duration_ms": 27.74, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4926,7 +4926,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.04, + "duration_ms": 28.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4954,7 +4954,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.52, + "duration_ms": 25.82, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4977,7 +4977,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.73, + "duration_ms": 26.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5004,7 +5004,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.71, + "duration_ms": 25.81, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5027,7 +5027,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.02, + "duration_ms": 25.75, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5056,7 +5056,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.32, + "duration_ms": 25.57, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5837,24 +5837,24 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "git_status": { "available": true, "dirty": true, - "changed_file_count": 51, + "changed_file_count": 34, "sample": [ " M registry/index.json", " M registry/packages/yao-meta-skill.json", - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/adoption_drift_report.json", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/context_budget.json", " M reports/context_budget.md", - " M reports/context_budget_summary.json" + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md" ], "scope": "generation-time status before this report is written" }, @@ -5864,9 +5864,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -5886,7 +5886,7 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, "public_claim": { "ready": false, @@ -5902,7 +5902,7 @@ }, "release_lock": { "ready": false, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "status_scope": "generation-time status before this report is written", "reason": "working tree was dirty at generation time" }, @@ -5912,7 +5912,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61" + "sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5986,7 +5986,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7966, - "sha256": "baf406c10747ebb03a12f0bc74233c3e98cf1715bebfcdb450286ac1b2f5277f" + "sha256": "874d573922462a290776489b7ab760136522b2780339b579e383ab8235af56e8" }, { "label": "blind_review", @@ -6021,7 +6021,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 134467, - "sha256": "875af23304269c89191e86f8c41e30cfbf5c9bf9603ae478c03bbc163b2a13ef" + "sha256": "d2e8352cae2c29a009d81bfc78ec4478cc36f045022d0803c7b229d6441fe1e9" }, { "label": "python_compatibility", @@ -6035,14 +6035,14 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "8707b96ff516f0e52201e9b2fa3fbbc59fdfb3046d7d5f534eae47c780563a93" + "sha256": "d0c088f66524fa16c604b88fcd45010303daed7eb9e5273e439451484c09442e" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "d8595fc2c41a897e607eae7a843bbdc28a2941b04d1e913ac9d2dbae233b7a2a" + "sha256": "d18b6336d485f6290b68bf229d491bcd8a08ad872a8e9a53502b996c4f9c1347" }, { "label": "install_simulation", @@ -6056,7 +6056,7 @@ "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14466, - "sha256": "b1c82626a1611c22c4d4cf3a6caa96a80355e9ecbdaee13a683456e648702406" + "sha256": "e6f644b1d2f590e76c7f2dd5a4888efad138bd2a2643c0fa60fe2fea81db55e1" }, { "label": "world_class_evidence_plan", @@ -6083,8 +6083,8 @@ "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 46188, - "sha256": "0c140e75d4146092b522ec65514d055c9e1cab9f543bc81b828f5a1b6ec712b8" + "bytes": 57313, + "sha256": "585ec2f93a1dfce5b270af048e3d551676b93231477d7bf4cfe5b08946e82619" }, { "label": "world_class_submission_review", @@ -12441,7 +12441,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "failures": [], "warnings": [], @@ -17318,6 +17318,14 @@ "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_world_class_preflight.py", + "lines": 590, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "tests/verify_review_studio.py", "lines": 590, @@ -17373,14 +17381,6 @@ "severity": "pass", "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, - { - "path": "scripts/render_review_studio.py", - "lines": 568, - "kind": "cli-script", - "severity": "pass", - "early_watch": false, - "recommendation": "Move data loading and large section renderers into focused review_studio_* modules." } ], "watchlist": [], @@ -17417,15 +17417,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1105271, + "other_text_tokens": 1107027, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1106068, - "deferred_resource_tokens": 503641, + "estimated_total_text_tokens": 1107824, + "deferred_resource_tokens": 504879, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148 }, { @@ -17447,7 +17447,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148 } ], @@ -17470,7 +17470,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 442611, + "estimated_tokens": 443849, "file_count": 148, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -21867,6 +21867,10 @@ "source_check_count": 19, "source_pass_count": 9, "source_blocked_count": 10, + "repair_checklist_count": 14, + "repair_blocked_count": 14, + "repair_ready_count": 0, + "repair_counts_as_completion": false, "pending_count": 4, "ready_to_claim_world_class": false, "credential_value_exposed": false, @@ -22030,6 +22034,38 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "openai-api-key", + "status": "blocked", + "blocking_reason": "Required env precheck is missing.", + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "model_executed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Run provider-backed output-exec with real credentials.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "token_observed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Provider execution should return non-estimated token usage.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -22247,6 +22283,68 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -22381,6 +22479,28 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -22514,6 +22634,38 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "source": "source_checklist", + "counts_as_completion": false + } ] } ], @@ -22878,6 +23030,148 @@ "next_action": "Telemetry must stay metadata-only." } ], + "repair_checklist": [ + { + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "openai-api-key", + "status": "blocked", + "blocking_reason": "Required env precheck is missing.", + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "model_executed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Run provider-backed output-exec with real credentials.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "token_observed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Provider execution should return non-estimated token usage.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "source": "source_checklist", + "counts_as_completion": false + } + ], "submissions": { "directory": "evidence/world_class/submissions", "commands": { @@ -24788,8 +25082,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -24820,7 +25114,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -24845,7 +25139,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, @@ -24868,7 +25162,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 @@ -25876,12 +26170,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index d5e7fcfa..cb9c0ef0 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -1010,9 +1010,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1032,9 +1032,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1139,7 +1139,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "skill_atlas": { "skill_count": 12, @@ -1177,8 +1177,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -1209,7 +1209,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1225,7 +1225,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 @@ -1304,12 +1304,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 42bccdd5..ce4b4e04 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 78072d36..e9f9bc79 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `240` -- Package SHA256: `6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e` +- Package SHA256: `cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242` ## Failures diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index f57bb3ce..80743cc1 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1014,9 +1014,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1036,9 +1036,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1143,7 +1143,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "skill_atlas": { "skill_count": 12, @@ -1181,8 +1181,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -1213,7 +1213,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1229,7 +1229,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 @@ -1308,12 +1308,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 2e99ce82..34023557 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -29,6 +29,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture. - World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. - World-Class Repair Checklist v0 so the submission kit turns every draft, artifact, and source-check blocker into machine-readable repair rows without counting repair guidance as completion evidence. +- World-Class Preflight Repair Rows v0 so the collection preflight exposes required precheck and source-check blockers as machine-readable operator actions before evidence submission. - World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. - World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. - Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. @@ -59,7 +60,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. - Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes. -This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, hook recipes now make Browser/Chrome/VS Code/wrapper/provider-adapter integration commands auditable, a Browser/Chrome Native Messaging host can receive length-prefixed metadata events and generate a local launcher plus manifest, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. World-class intake now rejects non-canonical submission filenames and nested raw content or credential-like fields before ledger review. The submission kit now gives operators a matrix and repair checklist that tie each pending world-class evidence key to its draft, artifact readiness, source blockers, and next action. Review Studio keeps pending human adjudication and pending world-class evidence visible as warnings instead of treating them as a clean pass. Deeper provider-native execution transforms, installed platform-native client telemetry implementations, provider-native installer integration, real provider holdout runs, real human adjudication decisions, accepted external evidence, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, hook recipes now make Browser/Chrome/VS Code/wrapper/provider-adapter integration commands auditable, a Browser/Chrome Native Messaging host can receive length-prefixed metadata events and generate a local launcher plus manifest, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. World-class intake now rejects non-canonical submission filenames and nested raw content or credential-like fields before ledger review. The preflight and submission kit now give operators repair rows, matrix state, artifact readiness, source blockers, and next actions for each pending world-class evidence key. Review Studio keeps pending human adjudication and pending world-class evidence visible as warnings instead of treating them as a clean pass. Deeper provider-native execution transforms, installed platform-native client telemetry implementations, provider-native installer integration, real provider holdout runs, real human adjudication decisions, accepted external evidence, and native runtime permission enforcement remain open. ## Coverage Matrix diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 76821d69..c2bb1fee 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -1009,9 +1009,9 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b40cc3b316af766a9b69286dd55f46ce1dfb41f2d3e2af811d34a3f57c5f61", - "source_contract_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "evidence_bundle_sha256": "fa79b95f174e1c2b41a83456e6f6d7d15064dfa388f5c2b79ccc12ccefa2d119", + "source_contract_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1031,9 +1031,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 51 + "changed_file_count": 34 }, - "commit": "2241b0e344e896e26a1b8264a797e45665613b8b", + "commit": "48a503f8f6ea9ed8b39c5f4c6c07adbc94bca3cb", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1138,7 +1138,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 240, - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" }, "skill_atlas": { "skill_count": 12, @@ -1176,8 +1176,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e", - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "package_sha256": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, "compatibility": { "openai": "pass", @@ -1208,7 +1208,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1224,7 +1224,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365", + "archive_sha256": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362", "archive_entry_count": 686, "failure_count": 0, "warning_count": 0 @@ -1303,12 +1303,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 8d244cf6..176c965f 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -307,7 +307,7 @@ "key": "review-studio", "label": "Review Studio", "status": "pass", - "current": "decision review; warnings 4; score 89", + "current": "decision review; warnings 3; score 91", "target": "One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 3d0afcae..276f5bbc 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -28,7 +28,7 @@ Generated at: `2026-06-17` | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | | Registry Distribution | pass | zip entries 686; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | -| Review Studio | pass | decision review; warnings 4; score 89 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | +| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index a5180c7e..c0595604 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "992401b48ffe5354fc67d493a44a5f6a2e93adf22d26b28d3ad291f5db813365" + "to": "b09ad63e1a0cb59beba3ba7f71e285e981b84b3de6d4f9f7404618c25d96c362" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6beb1e623996dc21a8b040d894e12a81fc4c70cc82641887f71c19933e34617e" + "to": "cc61aeda8a4a7567aca6432c73fc8faf0ed3ed753c8254245d92675edeec5242" } ] }, diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index 63b56324..fdfa4e94 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -21,10 +21,10 @@ h3 { margin:4px 0 10px; font-size:22px; letter-spacing:0; } h4 { margin:0 0 8px; font-size:16px; letter-spacing:0; } .lede { max-width:820px; color:var(--muted); font-size:20px; } - .stats { display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin:26px 0 0; } - .stats article, .panel, .evidence-card, .check-row { border:1px solid var(--line); border-radius:8px; background:#fff; } + .stats { display:grid; grid-template-columns:repeat(6, minmax(0,1fr)); gap:12px; margin:26px 0 0; } + .stats article, .panel, .evidence-card, .check-row, .repair-row { border:1px solid var(--line); border-radius:8px; background:#fff; } .stats article { padding:16px; } - .stats span, .muted, .evidence-card header span, .check-row span { color:var(--muted); } + .stats span, .muted, .evidence-card header span, .check-row span, .repair-row span { color:var(--muted); } .stats strong { display:block; color:var(--ink); font-size:28px; line-height:1.15; overflow-wrap:anywhere; } .section { padding:32px 0; border-bottom:1px solid var(--line); } .two-col { display:grid; grid-template-columns:minmax(0,.45fr) minmax(0,1fr); gap:18px; align-items:start; } @@ -41,9 +41,9 @@ .evidence-grid { display:grid; gap:18px; } .evidence-card { padding:20px; min-width:0; } .evidence-card.blocked { border-left:4px solid var(--block); } - .evidence-card.ready-for-human-review, .evidence-card.ready-to-collect, .check-row.human-required, .check-row.external-required, .check-row.missing, .check-row.blocked { border-left:4px solid var(--warn); } - .evidence-card.ready-for-submission, .check-row.pass { border-left:4px solid var(--pass); } - .meta, .check-row dl { display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; } + .evidence-card.ready-for-human-review, .evidence-card.ready-to-collect, .check-row.human-required, .check-row.external-required, .check-row.missing, .check-row.blocked, .repair-row.blocked { border-left:4px solid var(--warn); } + .evidence-card.ready-for-submission, .check-row.pass, .repair-row.ready { border-left:4px solid var(--pass); } + .meta, .check-row dl, .repair-row dl { display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; } dt { color:var(--ink); } dd { margin:0; min-width:0; overflow-wrap:anywhere; } code { font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:13px; overflow-wrap:anywhere; } @@ -51,11 +51,12 @@ .next-action p { margin-top:0; } .next-action code { display:block; margin-top:8px; } .check-grid { display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:12px; } - .check-row { padding:14px; min-width:0; } + .repair-grid { display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:12px; } + .check-row, .repair-row { padding:14px; min-width:0; } .check-section { margin-top:16px; } .notice { background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; } li { overflow-wrap:anywhere; } - @media (max-width:820px) { .stats, .two-col, .check-grid, .role-grid { grid-template-columns:1fr; } h1 { font-size:38px; } .topbar-inner { align-items:flex-start; flex-direction:column; } } + @media (max-width:820px) { .stats, .two-col, .check-grid, .repair-grid, .role-grid { grid-template-columns:1fr; } h1 { font-size:38px; } .topbar-inner { align-items:flex-start; flex-direction:column; } } @@ -65,7 +66,7 @@ Evidence Collection

World-Class Evidence Preflight

This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

-
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source9/19
+
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source9/19
Repairs14/14
@@ -232,6 +233,49 @@
+
+

Repair Rows

+
+
+
+ precheck + openai-api-key +
+
+
Status
blocked
+
Reason
Required env precheck is missing.
+
Action
Set OPENAI_API_KEY in the operator shell; never commit or print the value.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + model_executed_count +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Run provider-backed output-exec with real credentials.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + token_observed_count +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Provider execution should return non-estimated token usage.
+
Evidence
does not count as completion
+
+
+
+

Runbook

@@ -462,6 +506,88 @@
+
+

Repair Rows

+
+
+
+ precheck + human-reviewer +
+
+
Status
blocked
+
Reason
Required human precheck is human-required.
+
Action
Assign a real reviewer identity before claiming human adjudication.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + pending_count +
+
+
Status
blocked
+
Reason
Current value 5 does not satisfy '==0'.
+
Action
Record a reviewer choice and reason for every pair.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + judgment_count +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '==pair_count'.
+
Action
Every pair needs one valid human judgment.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + reviewer_metadata_present +
+
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Record reviewer and reviewed_at before adjudication can count.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + blind_review_attested +
+
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Set reviewer_attestation only after choices are completed before opening the answer key.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + ready_for_human_evidence +
+
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
+
Evidence
does not count as completion
+
+
+
+

Runbook

@@ -589,6 +715,36 @@
+
+

Repair Rows

+
+
+
+ precheck + native-guard +
+
+
Status
blocked
+
Reason
Required external precheck is external-required.
+
Action
Attach a real target-client or external installer runtime guard; metadata fallback is not enough.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + native_enforcement_count +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Collect real target-client or external runtime guard proof.
+
Evidence
does not count as completion
+
+
+
+

Runbook

@@ -716,6 +872,49 @@
+
+

Repair Rows

+
+
+
+ precheck + external-client +
+
+
Status
blocked
+
Reason
Required external precheck is external-required.
+
Action
Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + external_source_events +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Import at least one metadata-only event from a real client.
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + adoption_sample_count +
+
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Telemetry must include adoption outcome evidence.
+
Evidence
does not count as completion
+
+
+
+

Runbook

diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 81019823..75529ca6 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -16,6 +16,10 @@ "source_check_count": 19, "source_pass_count": 9, "source_blocked_count": 10, + "repair_checklist_count": 14, + "repair_blocked_count": 14, + "repair_ready_count": 0, + "repair_counts_as_completion": false, "pending_count": 4, "ready_to_claim_world_class": false, "credential_value_exposed": false, @@ -179,6 +183,38 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "openai-api-key", + "status": "blocked", + "blocking_reason": "Required env precheck is missing.", + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "model_executed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Run provider-backed output-exec with real credentials.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "token_observed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Provider execution should return non-estimated token usage.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -396,6 +432,68 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -530,6 +628,28 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "source": "source_checklist", + "counts_as_completion": false + } ] }, { @@ -663,6 +783,38 @@ "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "source": "source_checklist", + "counts_as_completion": false + } ] } ], @@ -1027,6 +1179,148 @@ "next_action": "Telemetry must stay metadata-only." } ], + "repair_checklist": [ + { + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "openai-api-key", + "status": "blocked", + "blocking_reason": "Required env precheck is missing.", + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "model_executed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Run provider-backed output-exec with real credentials.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "provider-holdout", + "repair_type": "source-check", + "target": "token_observed_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Provider execution should return non-estimated token usage.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "source": "prechecks", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "source": "source_checklist", + "counts_as_completion": false + } + ], "submissions": { "directory": "evidence/world_class/submissions", "commands": { diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index 962dba67..c4a1adbb 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -11,6 +11,7 @@ Generated at: `2026-06-17` - collection ready: `1` - collection blocked: `3` - source checks: `9` pass / `19` total +- repair rows: `14` blocked / `14` total This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger. @@ -46,6 +47,27 @@ Generate the submission kit after the real provider, human, native-permission, o | `native-permission-enforcement` | `blocked` | `awaiting-submission` | `awaiting-submission` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | | `native-client-telemetry` | `blocked` | `awaiting-submission` | `awaiting-submission` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | +## Repair Checklist + +Repair rows convert preflight and source blockers into concrete operator actions. They are guidance only and do not count as completion evidence. + +| Evidence | Type | Target | Status | Next action | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `precheck` | `openai-api-key` | `blocked` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | +| `provider-holdout` | `source-check` | `model_executed_count` | `blocked` | Run provider-backed output-exec with real credentials. | +| `provider-holdout` | `source-check` | `token_observed_count` | `blocked` | Provider execution should return non-estimated token usage. | +| `human-adjudication` | `precheck` | `human-reviewer` | `blocked` | Assign a real reviewer identity before claiming human adjudication. | +| `human-adjudication` | `source-check` | `pending_count` | `blocked` | Record a reviewer choice and reason for every pair. | +| `human-adjudication` | `source-check` | `judgment_count` | `blocked` | Every pair needs one valid human judgment. | +| `human-adjudication` | `source-check` | `reviewer_metadata_present` | `blocked` | Record reviewer and reviewed_at before adjudication can count. | +| `human-adjudication` | `source-check` | `blind_review_attested` | `blocked` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| `human-adjudication` | `source-check` | `ready_for_human_evidence` | `blocked` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | +| `native-permission-enforcement` | `precheck` | `native-guard` | `blocked` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | +| `native-permission-enforcement` | `source-check` | `native_enforcement_count` | `blocked` | Collect real target-client or external runtime guard proof. | +| `native-client-telemetry` | `precheck` | `external-client` | `blocked` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | +| `native-client-telemetry` | `source-check` | `external_source_events` | `blocked` | Import at least one metadata-only event from a real client. | +| `native-client-telemetry` | `source-check` | `adoption_sample_count` | `blocked` | Telemetry must include adoption outcome evidence. | + ## Provider Holdout - status: `blocked` diff --git a/scripts/render_world_class_preflight.py b/scripts/render_world_class_preflight.py index 76865c1e..114e186e 100644 --- a/scripts/render_world_class_preflight.py +++ b/scripts/render_world_class_preflight.py @@ -13,6 +13,7 @@ from render_world_class_evidence_ledger import build_ledger from render_world_class_submission_review import build_submission_review from world_class_evidence_contract import rel_path from world_class_preflight_layout import render_html +from world_class_repair_checklist import build_preflight_repair_checklist, summarize_repair_checklist from world_class_source_checks import summarize_source_checklist @@ -346,11 +347,19 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | items.append(item) precheck_rows.extend(prechecks) source_rows.extend(item_source_rows) + repair_checklist = build_preflight_repair_checklist(items) + repairs_by_key: dict[str, list[dict[str, Any]]] = {} + for row in repair_checklist: + key = str(row.get("evidence_key", "")) + repairs_by_key.setdefault(key, []).append(row) + for item in items: + item["repair_checklist"] = repairs_by_key.get(str(item.get("evidence_key", "")), []) precheck_status_counts: dict[str, int] = {} for row in precheck_rows: status = str(row.get("status", "unknown")) precheck_status_counts[status] = precheck_status_counts.get(status, 0) + 1 source_summary = summarize_source_checklist(source_rows) + repair_summary = summarize_repair_checklist(repair_checklist) collection_ready_count = sum(1 for item in items if item["status"] in {"ready-to-collect", "ready-for-human-review", "ready-for-submission"}) blocked_count = sum(1 for item in items if item["status"] == "blocked") ready_to_claim = ledger.get("summary", {}).get("ready_to_claim_world_class") is True @@ -365,6 +374,7 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | "collection_ready_count": collection_ready_count, "collection_blocked_count": blocked_count, **source_summary, + **repair_summary, "pending_count": ledger.get("summary", {}).get("pending_count", 0), "ready_to_claim_world_class": ready_to_claim, "credential_value_exposed": False, @@ -382,6 +392,7 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | "items": items, "prechecks": precheck_rows, "source_checklist": source_rows, + "repair_checklist": repair_checklist, "submissions": { "directory": rel_path(submissions_dir, skill_dir), "commands": build_submission_commands(skill_dir, submissions_dir), @@ -425,6 +436,7 @@ def render_markdown(report: dict[str, Any]) -> str: f"- collection ready: `{summary['collection_ready_count']}`", f"- collection blocked: `{summary['collection_blocked_count']}`", f"- source checks: `{summary['source_pass_count']}` pass / `{summary['source_check_count']}` total", + f"- repair rows: `{summary['repair_blocked_count']}` blocked / `{summary['repair_checklist_count']}` total", "", "This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.", "", @@ -475,6 +487,22 @@ def render_markdown(report: dict[str, Any]) -> str: lines.append( f"| `{item['evidence_key']}` | `{item['status']}` | `{item['intake_readiness']}` | `{item['review_state']}` | {next_action} |" ) + lines.extend( + [ + "", + "## Repair Checklist", + "", + "Repair rows convert preflight and source blockers into concrete operator actions. They are guidance only and do not count as completion evidence.", + "", + "| Evidence | Type | Target | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in report.get("repair_checklist", []): + lines.append( + f"| `{row['evidence_key']}` | `{row['repair_type']}` | `{row['target']}` | " + f"`{row['status']}` | {md_cell(row['next_action'])} |" + ) for item in report["items"]: lines.extend( [ diff --git a/scripts/world_class_preflight_layout.py b/scripts/world_class_preflight_layout.py index 586f4a1a..d33c8346 100644 --- a/scripts/world_class_preflight_layout.py +++ b/scripts/world_class_preflight_layout.py @@ -79,6 +79,34 @@ def render_html_source_checks(rows: list[dict[str, Any]]) -> str: ) +def render_html_repair_rows(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

No repair rows listed.

" + return "".join( + """ +
+
+ {repair_type} + {target} +
+
+
Status
{status}
+
Reason
{reason}
+
Action
{action}
+
Evidence
does not count as completion
+
+
+ """.format( + status=html_text(row.get("status", "")), + repair_type=html_text(row.get("repair_type", "")), + target=html_text(row.get("target", "")), + reason=html_text(row.get("blocking_reason", "")), + action=html_text(row.get("next_action", "")), + ) + for row in rows + ) + + def render_html_artifact_roles(contract: dict[str, Any]) -> str: cards = [] for role in contract.get("roles", []): @@ -143,6 +171,10 @@ def render_html_item(item: dict[str, Any]) -> str:

Source Checks

{render_html_source_checks(item.get('source_checklist', []))}
+
+

Repair Rows

+
{render_html_repair_rows(item.get('repair_checklist', []))}
+

Runbook

@@ -159,6 +191,7 @@ def render_html(report: dict[str, Any]) -> str: ("Ready", summary["collection_ready_count"]), ("Blocked", summary["collection_blocked_count"]), ("Source", f"{summary['source_pass_count']}/{summary['source_check_count']}"), + ("Repairs", f"{summary.get('repair_blocked_count', 0)}/{summary.get('repair_checklist_count', 0)}"), ] stat_html = "".join( f"
{html_text(label)}{html_text(value)}
" @@ -189,10 +222,10 @@ def render_html(report: dict[str, Any]) -> str: h3 {{ margin:4px 0 10px; font-size:22px; letter-spacing:0; }} h4 {{ margin:0 0 8px; font-size:16px; letter-spacing:0; }} .lede {{ max-width:820px; color:var(--muted); font-size:20px; }} - .stats {{ display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin:26px 0 0; }} - .stats article, .panel, .evidence-card, .check-row {{ border:1px solid var(--line); border-radius:8px; background:#fff; }} + .stats {{ display:grid; grid-template-columns:repeat(6, minmax(0,1fr)); gap:12px; margin:26px 0 0; }} + .stats article, .panel, .evidence-card, .check-row, .repair-row {{ border:1px solid var(--line); border-radius:8px; background:#fff; }} .stats article {{ padding:16px; }} - .stats span, .muted, .evidence-card header span, .check-row span {{ color:var(--muted); }} + .stats span, .muted, .evidence-card header span, .check-row span, .repair-row span {{ color:var(--muted); }} .stats strong {{ display:block; color:var(--ink); font-size:28px; line-height:1.15; overflow-wrap:anywhere; }} .section {{ padding:32px 0; border-bottom:1px solid var(--line); }} .two-col {{ display:grid; grid-template-columns:minmax(0,.45fr) minmax(0,1fr); gap:18px; align-items:start; }} @@ -209,9 +242,9 @@ def render_html(report: dict[str, Any]) -> str: .evidence-grid {{ display:grid; gap:18px; }} .evidence-card {{ padding:20px; min-width:0; }} .evidence-card.blocked {{ border-left:4px solid var(--block); }} - .evidence-card.ready-for-human-review, .evidence-card.ready-to-collect, .check-row.human-required, .check-row.external-required, .check-row.missing, .check-row.blocked {{ border-left:4px solid var(--warn); }} - .evidence-card.ready-for-submission, .check-row.pass {{ border-left:4px solid var(--pass); }} - .meta, .check-row dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }} + .evidence-card.ready-for-human-review, .evidence-card.ready-to-collect, .check-row.human-required, .check-row.external-required, .check-row.missing, .check-row.blocked, .repair-row.blocked {{ border-left:4px solid var(--warn); }} + .evidence-card.ready-for-submission, .check-row.pass, .repair-row.ready {{ border-left:4px solid var(--pass); }} + .meta, .check-row dl, .repair-row dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }} dt {{ color:var(--ink); }} dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }} code {{ font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:13px; overflow-wrap:anywhere; }} @@ -219,11 +252,12 @@ def render_html(report: dict[str, Any]) -> str: .next-action p {{ margin-top:0; }} .next-action code {{ display:block; margin-top:8px; }} .check-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:12px; }} - .check-row {{ padding:14px; min-width:0; }} + .repair-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:12px; }} + .check-row, .repair-row {{ padding:14px; min-width:0; }} .check-section {{ margin-top:16px; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} li {{ overflow-wrap:anywhere; }} - @media (max-width:820px) {{ .stats, .two-col, .check-grid, .role-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} + @media (max-width:820px) {{ .stats, .two-col, .check-grid, .repair-grid, .role-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} diff --git a/scripts/world_class_repair_checklist.py b/scripts/world_class_repair_checklist.py index b243c0d2..04571e60 100644 --- a/scripts/world_class_repair_checklist.py +++ b/scripts/world_class_repair_checklist.py @@ -144,6 +144,56 @@ def build_repair_checklist( return rows +def build_preflight_repair_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return repair rows from preflight precheck and source-check blockers.""" + rows: list[dict[str, Any]] = [] + for item in items: + key = str(item.get("evidence_key", "")).strip() + if not key: + continue + for precheck in item.get("prechecks", []): + if not isinstance(precheck, dict): + continue + if precheck.get("required") is not True or precheck.get("status") == "pass": + continue + target = str(precheck.get("key") or precheck.get("label") or key) + rows.append( + _repair_row( + evidence_key=key, + repair_type="precheck", + target=target, + status="blocked", + blocking_reason=( + f"Required {precheck.get('kind', 'precheck')} precheck is " + f"{precheck.get('status', 'unknown')}." + ), + next_action=str(precheck.get("next_action") or "Complete the required preflight action."), + source="prechecks", + ) + ) + for source in item.get("source_checklist", []): + if not isinstance(source, dict): + continue + if source.get("status") == "pass": + continue + target = str(source.get("field") or source.get("label") or key) + rows.append( + _repair_row( + evidence_key=key, + repair_type="source-check", + target=target, + status="blocked", + blocking_reason=( + f"Current value {source.get('actual')!r} does not satisfy " + f"{source.get('expected')!r}." + ), + next_action=str(source.get("next_action") or "Collect the required source evidence."), + source="source_checklist", + ) + ) + return rows + + def summarize_repair_checklist(rows: list[dict[str, Any]]) -> dict[str, Any]: blocked_count = sum(1 for row in rows if row.get("status") != "ready") return { diff --git a/tests/verify_world_class_preflight.py b/tests/verify_world_class_preflight.py index 33e3873b..31dd00c6 100644 --- a/tests/verify_world_class_preflight.py +++ b/tests/verify_world_class_preflight.py @@ -79,6 +79,15 @@ def main() -> None: assert summary["source_check_count"] >= 13, summary assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary assert summary["source_blocked_count"] >= 6, summary + assert summary["repair_checklist_count"] == ( + summary["source_blocked_count"] + + summary["precheck_missing_count"] + + summary["precheck_human_required_count"] + + summary["precheck_external_required_count"] + ), summary + assert summary["repair_blocked_count"] == summary["repair_checklist_count"], summary + assert summary["repair_ready_count"] == 0, summary + assert summary["repair_counts_as_completion"] is False, summary assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload assert payload["submissions"]["drafts_count_as_evidence"] is False, payload assert payload["submissions"]["submission_kit_command"] == ( @@ -147,6 +156,11 @@ def main() -> None: assert "sk-test-secret" not in proc.stdout, proc.stdout assert "OPENAI_API_KEY" in proc.stdout, proc.stdout assert "set OPENAI_API_KEY" not in proc.stdout.lower(), proc.stdout + provider_repairs = {item["target"]: item for item in provider["repair_checklist"]} + assert provider_repairs["openai-api-key"]["repair_type"] == "precheck", provider_repairs + assert provider_repairs["openai-api-key"]["counts_as_completion"] is False, provider_repairs + assert provider_repairs["model_executed_count"]["repair_type"] == "source-check", provider_repairs + assert provider_repairs["token_observed_count"]["status"] == "blocked", provider_repairs human = by_key(payload["items"], "human-adjudication") assert human["status"] == "ready-for-human-review", human @@ -156,6 +170,9 @@ def main() -> None: assert any("Record a reviewer choice and reason" in row["next_action"] for row in human["source_checklist"]), human assert any("required rationale" in item["next_action"] for item in human["prechecks"]), human assert any("reviewed_at" in item["next_action"] for item in human["prechecks"]), human + human_repairs = {item["target"]: item for item in human["repair_checklist"]} + assert human_repairs["human-reviewer"]["repair_type"] == "precheck", human_repairs + assert human_repairs["pending_count"]["repair_type"] == "source-check", human_repairs native = by_key(payload["items"], "native-permission-enforcement") assert native["status"] == "blocked", native @@ -167,6 +184,11 @@ def main() -> None: assert "preflight counts as evidence: `false`" in markdown, markdown assert "credential value exposed: `false`" in markdown, markdown assert "Submission Kit Handoff" in markdown, markdown + assert "Repair Checklist" in markdown, markdown + assert "Repair rows convert preflight and source blockers" in markdown, markdown + assert "repair rows:" in markdown, markdown + assert "`openai-api-key`" in markdown, markdown + assert "`model_executed_count`" in markdown, markdown assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in markdown, markdown assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in markdown, markdown assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" in markdown, markdown @@ -183,6 +205,11 @@ def main() -> None: assert "World-Class Evidence Preflight" in html, html assert "Evidence Queue" in html, html assert "Submission Kit" in html, html + assert "Repair Rows" in html, html + assert "Repairs" in html, html + assert "repair-row blocked" in html, html + assert "openai-api-key" in html, html + assert "model_executed_count" in html, html assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in html, html assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in html, html assert "provider-holdout" in html, html @@ -215,6 +242,7 @@ def main() -> None: assert "sk-test-secret" not in (TMP / "preflight_with_env.html").read_text(encoding="utf-8"), env_payload assert env_payload["summary"]["credential_value_exposed"] is False, env_payload assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload + assert env_payload["summary"]["repair_checklist_count"] == payload["summary"]["repair_checklist_count"] - 1, env_payload spaced_dir = TMP / "submission kit spaced" spaced_proc = run_preflight(