diff --git a/evidence/world_class/README.md b/evidence/world_class/README.md index d13b1782..bc05fa69 100644 --- a/evidence/world_class/README.md +++ b/evidence/world_class/README.md @@ -34,6 +34,8 @@ The generated intake report also includes an `operator_checklist` for each pendi The submission kit command creates editable JSON drafts plus a local README for an external operator or human reviewer. Use `--prefill-artifacts` when you want the kit to insert SHA-256 digests for currently available local aggregate artifacts. Prefill is operator convenience only: those drafts still keep `template_only: true` and do not count as evidence until the real run or review exists, the packet is edited truthfully, every artifact ref points to a local aggregate evidence file with a matching `sha256`, and `world-class-intake` validates it. +The submission kit separates artifact rows into `submission-ref` and `supporting-evidence`. `submission-ref` rows are the concrete paths expected in a real packet's `artifact_refs`; `supporting-evidence` rows help reviewers audit the packet and do not all need to be copied into the submission. + The submission review command renders a read-only queue that compares valid packets with the source evidence checks and current ledger state. It is for reviewer triage only; it does not accept evidence or make the world-class claim true. Accepted intake means "ready for ledger review", not evidence that the final public claim is ready. The ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/registry/index.json b/registry/index.json index c76819f0..12fbe8c1 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 1fbf98e7..631175ee 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 4c185ceb..6bc524ea 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -36,6 +36,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 867, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_evidence_consistency.py", "lines": 859, @@ -43,13 +50,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 842, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", "lines": 816, @@ -122,6 +122,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 867, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_evidence_consistency.py", "lines": 859, @@ -129,13 +136,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 842, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", "lines": 816, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 3ab9682b..6be14d87 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -31,8 +31,8 @@ No file-size hotspots found. | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | -| `tests/verify_world_class_evidence_intake.py` | `842` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `tests/verify_review_studio.py` | `816` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -45,8 +45,8 @@ No file-size hotspots found. | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | `pass` | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `842` | `test` | `pass` | | `tests/verify_review_studio.py` | `816` | `test` | `pass` | | `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 24b4e592..06c0263d 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,23 +3,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 22, + "sample": [ + " M evidence/world_class/README.md", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/package_verification.json", + " M reports/package_verification.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "ready": false, + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b" + "sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -138,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "dc896e2929e25d8f517ded4024f2bce12030f4be14f885f76255be887117766c" + "sha256": "3cacfc88fbb221a0c3e736d2f70683bf45d221b17cfe81ec7be4c8375eab9b14" }, { "label": "blind_review", @@ -173,7 +187,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 117850, - "sha256": "8a6f0eb0c627b1b8a7ffb45fd431b678827a91590028021e145d6b60d2fc4c2e" + "sha256": "6273c1b82d09eda9eaed37cd2335e0a9d71b0de130fce5d0692f37021a92ce4b" }, { "label": "python_compatibility", @@ -187,14 +201,14 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "bd4b1f1cd5b21d0bda736ea14db4e6839e3729f66c787ef3be97619e8d069267" + "sha256": "de8f4a0257c67a0b605ecaba30fdb89d9a0c7ba5256dd975c1e66b1df884b6fd" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "3fe267c06ebba290d312575d3042fc8a1125ebe7c07c1c555eae2948dbf845d0" + "sha256": "5ca9869ad3666cbf6fa0b31ba97717117e240be0c927982808ce8bbeccf68eed" }, { "label": "install_simulation", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 11ce79f1..668afba2 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,19 +1,19 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `30451d41f615562dfd0380414cc8447d4dad5e38` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b` +Commit: `e0a19e9b24a28e57e04aa8254013402eb93bf86b` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `1b03b1e4bab9` -- archive sha256: `38e8e607c1ed` +- source contract sha256: `892708619b13` +- archive sha256: `b90b116cb657` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `23` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `22` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b` +- sha256: `cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0` ## Methodology Sections @@ -72,15 +73,15 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `dc896e2929e2` | +| output_execution | `reports/output_execution_runs.json` | present | `3cacfc88fbb2` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `8a6f0eb0c627` | +| trust_report | `reports/security_trust_report.json` | present | `6273c1b82d09` | | python_compatibility | `reports/python_compatibility.json` | present | `364e9bbc8dd4` | -| registry_audit | `reports/registry_audit.json` | present | `bd4b1f1cd5b2` | -| package_verification | `reports/package_verification.json` | present | `3fe267c06ebb` | +| registry_audit | `reports/registry_audit.json` | present | `de8f4a0257c6` | +| package_verification | `reports/package_verification.json` | present | `5ca9869ad366` | | install_simulation | `reports/install_simulation.json` | present | `0a127a000c33` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `de180d2b67b0` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `38e95161404d` | diff --git a/reports/context_budget.json b/reports/context_budget.json index 56194176..da4f7dcb 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1037197, + "other_text_tokens": 1039014, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1037994, - "deferred_resource_tokens": 478929, + "estimated_total_text_tokens": 1039811, + "deferred_resource_tokens": 480160, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index 4ff8d750..e845cc3d 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 478929 | `governed` | scripts:419118 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 480160 | `governed` | scripts:420349 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index fb75396d..0fbbe5fb 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 478929, + "deferred_resource_tokens": 480160, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 1bafc541..f7b2420c 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 478929/120000; top deferred scripts 419118; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 480160/120000; top deferred scripts 420349; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 478929/120000; top deferred scripts 419118; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 480160/120000; top deferred scripts 420349; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -207,8 +207,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -248,8 +248,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "30451d41f615562dfd0380414cc8447d4dad5e38", - "actual": "30451d41f615562dfd0380414cc8447d4dad5e38", + "expected": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", + "actual": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -261,30 +261,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -394,8 +394,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "30451d41f615562dfd0380414cc8447d4dad5e38", - "actual": "30451d41f615562dfd0380414cc8447d4dad5e38", + "expected": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", + "actual": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -407,30 +407,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 41f2ada6..a9d95da2 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.75, + "duration_ms": 27.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.99, + "duration_ms": 28.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.29, + "duration_ms": 28.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.72, + "duration_ms": 29.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.38, + "duration_ms": 29.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 37.41, + "duration_ms": 28.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.96, + "duration_ms": 27.07, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.52, + "duration_ms": 26.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.23, + "duration_ms": 26.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.38, + "duration_ms": 26.28, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 3db2c164..599a9786 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 26.75 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 25.99 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.29 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.72 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 29.38 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 37.41 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.96 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.52 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.23 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.38 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.55 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.21 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.18 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.22 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.62 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.61 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.07 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.04 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.26 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.28 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 3653848c..e7efcc85 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index 38993f02..7c6578ac 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca` +- Archive SHA256: `b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 8a45f874..940071e6 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index faaa67f2..f6493114 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60` -- Archive SHA256: `d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096` +- Package SHA256: `892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c` +- Archive SHA256: `38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 1e13bced..10f3d0e6 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -711,12 +711,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

121 scripts scanned; secrets found

Py Compat0

193 files scanned for Python 3.11

Arch Debt0

899 largest lines; 9 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

646 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

121 scripts scanned; secrets found

Py Compat0

193 files scanned for Python 3.11

Arch Debt0

899 largest lines; 9 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

646 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 478929/120000; top deferred scripts 419118; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 121 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 193 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

190 Python files; 0 hotspots; 9 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 480160/120000; top deferred scripts 420349; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 121 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 193 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

190 Python files; 0 hotspots; 9 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -778,12 +778,12 @@
-

上下文

initial load 990/1000; deferred 478929/120000; top deferred scripts 419118; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 480160/120000; top deferred scripts 420349; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
121
网络脚本
3
Help 失败
0
包体哈希
1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60
+

信任报告

Secret
0
脚本数
121
网络脚本
3
Help 失败
0
包体哈希
892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -848,8 +848,8 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
+

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
@@ -869,12 +869,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
646
失败数
0
警告数
0
归档哈希
38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
646
失败数
0
警告数
0
归档哈希
b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d
diff --git a/reports/review-studio.json b/reports/review-studio.json index 525a9e22..852d1cb9 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -43,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 478929/120000; top deferred scripts 419118; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 480160/120000; top deferred scripts 420349; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -1812,13 +1812,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1836,11 +1836,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1945,7 +1945,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "skill_atlas": { "skill_count": 12, @@ -1983,8 +1983,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -2015,7 +2015,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -2031,7 +2031,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 @@ -2110,12 +2110,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, @@ -4488,7 +4488,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.75, + "duration_ms": 27.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4516,7 +4516,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.99, + "duration_ms": 28.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4539,7 +4539,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.29, + "duration_ms": 28.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4567,7 +4567,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.72, + "duration_ms": 29.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4590,7 +4590,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.38, + "duration_ms": 29.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4618,7 +4618,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 37.41, + "duration_ms": 28.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4641,7 +4641,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.96, + "duration_ms": 27.07, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4668,7 +4668,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.52, + "duration_ms": 26.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4691,7 +4691,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.23, + "duration_ms": 26.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4720,7 +4720,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.38, + "duration_ms": 26.28, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5446,23 +5446,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 22, + "sample": [ + " M evidence/world_class/README.md", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/package_verification.json", + " M reports/package_verification.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -5480,14 +5493,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5496,10 +5510,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "ready": false, + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5507,7 +5521,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b" + "sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5581,7 +5595,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "dc896e2929e25d8f517ded4024f2bce12030f4be14f885f76255be887117766c" + "sha256": "3cacfc88fbb221a0c3e736d2f70683bf45d221b17cfe81ec7be4c8375eab9b14" }, { "label": "blind_review", @@ -5616,7 +5630,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 117850, - "sha256": "8a6f0eb0c627b1b8a7ffb45fd431b678827a91590028021e145d6b60d2fc4c2e" + "sha256": "6273c1b82d09eda9eaed37cd2335e0a9d71b0de130fce5d0692f37021a92ce4b" }, { "label": "python_compatibility", @@ -5630,14 +5644,14 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "bd4b1f1cd5b21d0bda736ea14db4e6839e3729f66c787ef3be97619e8d069267" + "sha256": "de8f4a0257c67a0b605ecaba30fdb89d9a0c7ba5256dd975c1e66b1df884b6fd" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "3fe267c06ebba290d312575d3042fc8a1125ebe7c07c1c555eae2948dbf845d0" + "sha256": "5ca9869ad3666cbf6fa0b31ba97717117e240be0c927982808ce8bbeccf68eed" }, { "label": "install_simulation", @@ -11891,7 +11905,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "failures": [], "warnings": [], @@ -16063,6 +16077,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 867, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_evidence_consistency.py", "lines": 859, @@ -16070,13 +16091,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 842, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", "lines": 816, @@ -16149,6 +16163,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 867, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_evidence_consistency.py", "lines": 859, @@ -16156,13 +16177,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 842, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", "lines": 816, @@ -16214,15 +16228,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1037197, + "other_text_tokens": 1039014, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1037994, - "deferred_resource_tokens": 478929, + "estimated_total_text_tokens": 1039811, + "deferred_resource_tokens": 480160, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121 }, { @@ -16244,7 +16258,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121 } ], @@ -16267,7 +16281,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 419118, + "estimated_tokens": 420349, "file_count": 121, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -22147,8 +22161,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -22179,7 +22193,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -22204,7 +22218,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, @@ -22227,7 +22241,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 @@ -23235,12 +23249,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 233ec052..57dde332 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -997,13 +997,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1021,11 +1021,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1130,7 +1130,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "skill_atlas": { "skill_count": 12, @@ -1168,8 +1168,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -1200,7 +1200,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1216,7 +1216,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 @@ -1295,12 +1295,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 0c16d1cd..72a57032 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index ea2d035b..5071f893 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `212` -- Package SHA256: `1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60` +- Package SHA256: `892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c` ## Failures diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 8cf8274a..ef4027b1 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1001,13 +1001,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1025,11 +1025,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1134,7 +1134,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "skill_atlas": { "skill_count": 12, @@ -1172,8 +1172,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -1204,7 +1204,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1220,7 +1220,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 @@ -1299,12 +1299,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index ed6b7353..ec965e3a 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -22,6 +22,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. - World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. +- World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. - World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. - Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 50ab80f3..d3ce9ed4 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -996,13 +996,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f7b6c8fac83208f863f1e110b175eac345a7da3dc76240fd4a027a8a5745cd8b", - "source_contract_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "evidence_bundle_sha256": "cb7f095c9e24168f804f6b04a4d30f6be1959f248f693510a57e55eaf842afa0", + "source_contract_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "30451d41f615562dfd0380414cc8447d4dad5e38", + "commit": "e0a19e9b24a28e57e04aa8254013402eb93bf86b", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1129,7 +1129,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 212, - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" }, "skill_atlas": { "skill_count": 12, @@ -1167,8 +1167,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60", - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "package_sha256": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, "compatibility": { "openai": "pass", @@ -1199,7 +1199,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096", + "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1215,7 +1215,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca", + "archive_sha256": "b90b116cb657d9fa1ba2f3f0620a05965890b996917626dcdc7f974a05aaeb0d", "archive_entry_count": 646, "failure_count": 0, "warning_count": 0 @@ -1294,12 +1294,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index b51ba0e6..9c0d7a8a 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "d543b93ac3b301b4abfb6f749509fac8234e501b2a5e2af93361a7250f8e5096" + "to": "38e8e607c1ed5bd016ea242f8fac3e8dfdfe16b165876bc4e19f587481e7c5ca" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "1b03b1e4bab90eac18aa1ee7f0ddd1d7293c3bdfb32620531c4fc231e33e2b60" + "to": "892708619b13ff1af2f189a9a4526123c509bb8b3899dc721af6e66a80ba492c" } ] }, diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index bd468599..ae28c9f4 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -69,6 +69,25 @@ def artifact_rows_by_key_and_path(rows: list[dict[str, Any]]) -> dict[tuple[str, return ready +def template_artifact_ref_paths(skill_dir: Path, item: dict[str, Any]) -> set[str]: + template_path = skill_dir / str(item.get("template_path", "")) + errors: list[str] = [] + payload = load_template_payload(template_path, errors) + if payload is None: + return set() + refs = payload.get("artifact_refs", []) + if not isinstance(refs, list): + return set() + paths: set[str] = set() + for ref in refs: + if not isinstance(ref, dict): + continue + path = str(ref.get("path", "")).strip() + if path: + paths.add(path) + return paths + + def load_template_payload(path: Path, errors: list[str]) -> dict[str, Any] | None: try: payload = json.loads(path.read_text(encoding="utf-8")) @@ -201,12 +220,22 @@ def artifact_row(skill_dir: Path, evidence_key: str, pattern: str, path: Path, s } +def with_artifact_role(row: dict[str, Any], submission_ref_paths: set[str]) -> dict[str, Any]: + is_submission_ref = str(row.get("path", "")) in submission_ref_paths + return { + **row, + "artifact_role": "submission-ref" if is_submission_ref else "supporting-evidence", + "submission_ref_required": is_submission_ref, + } + + def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[dict[str, Any]]: key = str(item.get("evidence_key", "")) must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} artifacts = must_collect.get("evidence_artifacts", []) if not isinstance(artifacts, list): return [] + submission_ref_paths = template_artifact_ref_paths(skill_dir, item) rows: list[dict[str, Any]] = [] for artifact in artifacts: pattern = str(artifact or "").strip() @@ -214,31 +243,12 @@ def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[d continue if Path(pattern).is_absolute() or ".." in Path(pattern).parts: rows.append( - { - "evidence_key": key, - "source_pattern": pattern, - "path": pattern, - "status": "unsafe-path", - "exists": False, - "is_file": False, - "sha256": "", - "artifact_ref_ready": False, - "copy_path": "", - "copy_sha256": "", - "contains_raw_content": False, - "concrete_reference_required": True, - } - ) - continue - if has_glob_pattern(pattern): - matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file()) - if not matches: - rows.append( + with_artifact_role( { "evidence_key": key, "source_pattern": pattern, "path": pattern, - "status": "glob-no-match", + "status": "unsafe-path", "exists": False, "is_file": False, "sha256": "", @@ -247,13 +257,38 @@ def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[d "copy_sha256": "", "contains_raw_content": False, "concrete_reference_required": True, - } + }, + submission_ref_paths, + ) + ) + continue + if has_glob_pattern(pattern): + matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file()) + if not matches: + rows.append( + with_artifact_role( + { + "evidence_key": key, + "source_pattern": pattern, + "path": pattern, + "status": "glob-no-match", + "exists": False, + "is_file": False, + "sha256": "", + "artifact_ref_ready": False, + "copy_path": "", + "copy_sha256": "", + "contains_raw_content": False, + "concrete_reference_required": True, + }, + submission_ref_paths, + ) ) continue for match in matches: - rows.append(artifact_row(skill_dir, key, pattern, match, "ready")) + rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, match, "ready"), submission_ref_paths)) continue - rows.append(artifact_row(skill_dir, key, pattern, skill_dir / pattern, "")) + rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, skill_dir / pattern, ""), submission_ref_paths)) return rows @@ -305,6 +340,10 @@ def build_submission_kit( artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready")) artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) + submission_ref_rows = [item for item in artifact_checklist if item.get("submission_ref_required")] + supporting_rows = [item for item in artifact_checklist if not item.get("submission_ref_required")] + submission_ref_ready_count = sum(1 for item in submission_ref_rows if item.get("artifact_ref_ready")) + supporting_artifact_ready_count = sum(1 for item in supporting_rows if item.get("artifact_ref_ready")) source_summary = summarize_source_checklist(source_checklist) matrix_summary = summarize_evidence_matrix(evidence_matrix) ok = not unknown_keys and skipped_count == 0 @@ -325,6 +364,12 @@ def build_submission_kit( "artifact_ready_count": artifact_ready_count, "artifact_missing_count": artifact_missing_count, "artifact_glob_expansion_count": artifact_glob_count, + "submission_ref_count": len(submission_ref_rows), + "submission_ref_ready_count": submission_ref_ready_count, + "submission_ref_missing_count": len(submission_ref_rows) - submission_ref_ready_count, + "supporting_artifact_count": len(supporting_rows), + "supporting_artifact_ready_count": supporting_artifact_ready_count, + "supporting_artifact_missing_count": len(supporting_rows) - supporting_artifact_ready_count, "artifact_prefill_enabled": prefill_artifacts, "artifact_ref_prefill_count": prefilled_artifact_ref_count, "artifact_ref_unfilled_count": unfilled_artifact_ref_count, diff --git a/scripts/world_class_submission_kit_rendering.py b/scripts/world_class_submission_kit_rendering.py index 8163169c..31074741 100644 --- a/scripts/world_class_submission_kit_rendering.py +++ b/scripts/world_class_submission_kit_rendering.py @@ -50,14 +50,15 @@ def render_readme(report: dict[str, Any]) -> str: "", "This matrix combines draft, artifact, and source-check readiness into one operator action list. Matrix rows are guidance only; they do not count as completion evidence.", "", - "| Evidence | Stage | Draft | Artifacts | Source checks | Next action |", - "| --- | --- | --- | ---: | ---: | --- |", + "| Evidence | Stage | Draft | Submission refs | Supporting assets | Source checks | Next action |", + "| --- | --- | --- | ---: | ---: | ---: | --- |", ] ) for item in report.get("evidence_matrix", []): lines.append( f"| `{item['evidence_key']}` | `{item['stage']}` | `{item['draft_status']}` | " - f"`{item['artifact_ready_count']}/{item['artifact_total_count']}` | " + f"`{item.get('submission_ref_ready_count', 0)}/{item.get('submission_ref_total_count', 0)}` | " + f"`{item.get('supporting_artifact_ready_count', 0)}/{item.get('supporting_artifact_total_count', 0)}` | " f"`{item['source_pass_count']}/{item['source_check_count']}` | {item['next_action']} |" ) lines.extend(["", "## Execution Runbook", ""]) @@ -77,13 +78,16 @@ def render_readme(report: dict[str, Any]) -> str: "", "Use these paths and SHA-256 digests when filling `artifact_refs`. Glob patterns are expanded into concrete files; submissions must reference concrete paths, not globs.", "", - "| Evidence | Path | Status | SHA-256 |", - "| --- | --- | --- | --- |", + "| Evidence | Role | Path | Status | SHA-256 |", + "| --- | --- | --- | --- | --- |", ] ) for item in report.get("artifact_checklist", []): digest = item.get("sha256") or "n/a" - lines.append(f"| `{item['evidence_key']}` | `{item['path']}` | `{item['status']}` | `{digest}` |") + lines.append( + f"| `{item['evidence_key']}` | `{item.get('artifact_role', 'supporting-evidence')}` | " + f"`{item['path']}` | `{item['status']}` | `{digest}` |" + ) lines.extend( [ "", @@ -172,6 +176,7 @@ def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str:
Pattern
{pattern}
+
Role
{role}
Status
{status}
SHA-256
{sha}
@@ -181,6 +186,7 @@ def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str: key=html_text(item.get("evidence_key", "")), path=html_text(item.get("path", "")), pattern=html_text(item.get("source_pattern", "")), + role=html_text(item.get("artifact_role", "supporting-evidence")), sha=html_text(item.get("sha256") or "n/a"), ) for item in items @@ -229,7 +235,8 @@ def render_html_matrix(items: list[dict[str, Any]]) -> str:
Draft
{draft}
-
Artifacts
{artifact_ready}/{artifact_total} ready
+
Submission refs
{submission_ref_ready}/{submission_ref_total} ready
+
Supporting assets
{supporting_ready}/{supporting_total} ready
Source
{source_pass}/{source_total} pass
Owner
{owner}
@@ -239,8 +246,10 @@ def render_html_matrix(items: list[dict[str, Any]]) -> str: stage=html_text(item.get("stage", "")), key=html_text(item.get("evidence_key", "")), draft=html_text(item.get("draft_status", "")), - artifact_ready=html_text(item.get("artifact_ready_count", 0)), - artifact_total=html_text(item.get("artifact_total_count", 0)), + submission_ref_ready=html_text(item.get("submission_ref_ready_count", 0)), + submission_ref_total=html_text(item.get("submission_ref_total_count", 0)), + supporting_ready=html_text(item.get("supporting_artifact_ready_count", 0)), + supporting_total=html_text(item.get("supporting_artifact_total_count", 0)), source_pass=html_text(item.get("source_pass_count", 0)), source_total=html_text(item.get("source_check_count", 0)), owner=html_text(item.get("owner", "")), @@ -291,12 +300,15 @@ def render_html(report: dict[str, Any]) -> str: summary = report["summary"] artifact_ready = summary.get("artifact_ready_count", 0) artifact_total = summary.get("artifact_checklist_count", 0) + submission_ref_ready = summary.get("submission_ref_ready_count", 0) + submission_ref_total = summary.get("submission_ref_count", 0) stats = [ ("Requested", summary["requested_count"]), ("Written", summary["written_count"]), ("Existing", summary["existing_count"]), ("Skipped", summary["skipped_count"]), - ("Artifacts", f"{artifact_ready}/{artifact_total}"), + ("Submit refs", f"{submission_ref_ready}/{submission_ref_total}"), + ("Support", f"{artifact_ready - submission_ref_ready}/{artifact_total - submission_ref_total}"), ("Prefilled", summary.get("artifact_ref_prefill_count", 0)), ] stat_html = "".join( @@ -378,9 +390,9 @@ def render_html(report: dict[str, Any]) -> str:

Workflow

  1. Run the real provider, human review, native permission, or native client telemetry work first.
  2. Edit the matching JSON draft with aggregate artifact references and provenance metadata.
  3. Set template_only to false only after real evidence exists.
  4. Use prefilled SHA-256 values as convenience data, not evidence acceptance.
  5. Validate intake, refresh the ledger, then guard public claims.
-

Evidence Matrix

The matrix combines draft status, artifact readiness, source checks, and the next operator action. It is guidance only and never counts as accepted evidence.

{matrix_html}
+

Evidence Matrix

The matrix separates submission artifact_refs from supporting assets, then combines draft status, source checks, and the next operator action. It is guidance only and never counts as accepted evidence.

{matrix_html}

Drafts

{render_html_files(report['files'])}
-

Artifact Checklist

Copy concrete paths and SHA-256 digests from here into artifact_refs after real evidence exists. Glob patterns are expanded for operator convenience only.

{artifact_html}
+

Artifact Checklist

Rows marked submission-ref are the paths expected in artifact_refs. Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into the submission. Glob patterns are expanded for operator convenience only.

{artifact_html}

Source Evidence Snapshot

This section shows current aggregate source checks. It explains remaining blockers without changing the ledger.

{source_html}

Evidence Requirements

{evidence_html}

Safety Boundary

diff --git a/scripts/world_class_submission_matrix.py b/scripts/world_class_submission_matrix.py index 42bb7603..943b5ef7 100644 --- a/scripts/world_class_submission_matrix.py +++ b/scripts/world_class_submission_matrix.py @@ -93,6 +93,10 @@ def build_evidence_matrix( draft_status, draft_path = _draft_status(files_by_key, key) artifact_ready_count = sum(1 for row in artifact_rows if row.get("artifact_ref_ready")) artifact_missing_count = len(artifact_rows) - artifact_ready_count + submission_ref_rows = [row for row in artifact_rows if row.get("submission_ref_required")] + supporting_rows = [row for row in artifact_rows if not row.get("submission_ref_required")] + submission_ref_ready_count = sum(1 for row in submission_ref_rows if row.get("artifact_ref_ready")) + supporting_artifact_ready_count = sum(1 for row in supporting_rows if row.get("artifact_ref_ready")) source_pass_count = sum(1 for row in source_rows if row.get("status") == "pass") source_blocked_count = len(source_rows) - source_pass_count invalid_draft = draft_status not in {"written", "exists"} @@ -115,6 +119,12 @@ def build_evidence_matrix( "artifact_ready_count": artifact_ready_count, "artifact_total_count": len(artifact_rows), "artifact_missing_count": artifact_missing_count, + "submission_ref_ready_count": submission_ref_ready_count, + "submission_ref_total_count": len(submission_ref_rows), + "submission_ref_missing_count": len(submission_ref_rows) - submission_ref_ready_count, + "supporting_artifact_ready_count": supporting_artifact_ready_count, + "supporting_artifact_total_count": len(supporting_rows), + "supporting_artifact_missing_count": len(supporting_rows) - supporting_artifact_ready_count, "source_pass_count": source_pass_count, "source_check_count": len(source_rows), "source_blocked_count": source_blocked_count, @@ -137,5 +147,15 @@ def summarize_evidence_matrix(matrix: list[dict[str, Any]]) -> dict[str, int]: "evidence_matrix_prepare_draft_count": sum(1 for item in matrix if item.get("stage") == "prepare-draft"), "evidence_matrix_fix_artifacts_count": sum(1 for item in matrix if item.get("stage") == "fix-artifacts"), "evidence_matrix_validate_packet_count": sum(1 for item in matrix if item.get("stage") == "validate-packet"), + "evidence_matrix_submission_ref_count": sum(int(item.get("submission_ref_total_count", 0)) for item in matrix), + "evidence_matrix_submission_ref_ready_count": sum( + int(item.get("submission_ref_ready_count", 0)) for item in matrix + ), + "evidence_matrix_supporting_artifact_count": sum( + int(item.get("supporting_artifact_total_count", 0)) for item in matrix + ), + "evidence_matrix_supporting_artifact_ready_count": sum( + int(item.get("supporting_artifact_ready_count", 0)) for item in matrix + ), "evidence_matrix_counts_as_completion": 0, } diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index f57d8264..4ef7a785 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -617,11 +617,19 @@ def main() -> None: assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"] assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] @@ -634,6 +642,10 @@ def main() -> None: assert matrix_row["draft_status"] == "written", matrix_row assert matrix_row["artifact_ready_count"] >= 1, matrix_row assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row + assert matrix_row["submission_ref_ready_count"] == 1, matrix_row + assert matrix_row["submission_ref_total_count"] == 1, matrix_row + assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row + assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row assert matrix_row["source_pass_count"] == 1, matrix_row assert matrix_row["source_check_count"] == 3, matrix_row assert matrix_row["source_blocked_count"] == 2, matrix_row @@ -642,8 +654,12 @@ def main() -> None: artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows source_rows = {item["field"]: item for item in kit_payload["source_checklist"]} assert source_rows["model_executed_count"]["status"] == "blocked", source_rows assert source_rows["model_executed_count"]["actual"] == 0, source_rows @@ -667,6 +683,10 @@ def main() -> None: assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme assert "Evidence Matrix" in kit_readme, kit_readme + assert "Submission refs" in kit_readme, kit_readme + assert "Supporting assets" in kit_readme, kit_readme + assert "`submission-ref`" in kit_readme, kit_readme + assert "`supporting-evidence`" in kit_readme, kit_readme assert "`collect-source`" in kit_readme, kit_readme assert "Matrix rows are guidance only" in kit_readme, kit_readme assert "Source Evidence Snapshot" in kit_readme, kit_readme @@ -680,7 +700,11 @@ def main() -> None: assert "Evidence Matrix" in kit_html, kit_html assert "matrix-card collect-source" in kit_html, kit_html assert ( - f"
Artifacts
{matrix_row['artifact_ready_count']}/{matrix_row['artifact_total_count']} ready
" + f"
Submission refs
{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready
" + in kit_html + ), kit_html + assert ( + f"
Supporting assets
{matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready
" in kit_html ), kit_html assert "
Source
1/3 pass
" in kit_html, kit_html @@ -692,6 +716,7 @@ def main() -> None: assert "Execution Runbook" in kit_html, kit_html assert "output-exec --provider-runner openai" in kit_html, kit_html assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html + assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html prefilled_kit_dir = TMP / "prefilled_submission_kit" prefilled_proc = run_kit(