diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 4cba4db3..a9331956 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,22 +3,35 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "61898d145c9ca6a20a27faa28708781625757b86", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 32, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/evidence_consistency.json", + " M reports/install_simulation.json", + " M reports/install_simulation.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "55b50f5f8c8f4f305f9930044f132bde0b216f6556febcb5b530655b17b64702", - "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", + "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, "failure_disclosure_count": 3, @@ -33,30 +46,31 @@ "world_class_open_gap_count": 4, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 32 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (8/16 pass, 8 blocked)" + "world-class source checks are not all accepted (9/17 pass, 8 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "61898d145c9ca6a20a27faa28708781625757b86", + "ready": false, + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "55b50f5f8c8f4f305f9930044f132bde0b216f6556febcb5b530655b17b64702" + "sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -137,8 +151,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7964, - "sha256": "282122e93b08d73b0c199715c51bc619a95af47d315047fc41dceb31f56a3293" + "bytes": 7963, + "sha256": "ce151bc7d860840d0308731c3cf0342cae0610489d4205b452bbd4e47cc607ef" }, { "label": "blind_review", @@ -151,8 +165,8 @@ "label": "review_adjudication", "path": "reports/output_review_adjudication.json", "exists": true, - "bytes": 11128, - "sha256": "4843590cec50718ca26b6d4817ba3b9cf89eb741bbac52472246224d045c4bde" + "bytes": 11102, + "sha256": "2ed79f8bc7a31059ce491d07b4250e8b6f57fefdddbcf2d60ded459fd3fc6102" }, { "label": "trigger_scorecard", @@ -173,21 +187,21 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 133860, - "sha256": "d865e5cc97a3b92e1332366c5c780354afb46038ba1c6ef152a61cc8407a19bc" + "sha256": "f110df76d439c01b2d9b4ea82561d6c5277ff407f7ef616504e3923438d4d96b" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 29196, - "sha256": "f8e498d295db139c49e2ea51bd2cbdd52db61fa2b0bd008ef74135601d691895" + "sha256": "636583616dbcc15fe06d582ff0eda17c08a3d37eae618b03e1ca9c666c5276d2" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "88c94a496b941ef0538a7bab550c85d2fc07487edf35e4cd39bde14b1475108c" + "sha256": "c28991bdf087bb31370774c1dcce9987ed41eac46b2415eecda092692b9dd915" }, { "label": "package_verification", @@ -200,71 +214,71 @@ "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "044b2cf73dd4559a2a0b2f6cbb5560677d409d4716d135261fac6399012d1abb" + "bytes": 8758, + "sha256": "c91884a915384363d4ea0339de5c9a3e37ede4356ad1069edf0ae6f880366cc7" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "e63bc60e69cdc0e613983edea2a66f6c80d0e4da92efd6f90203c390b813350d" + "sha256": "4f193d15332d6ae7c6df397b0c777219fc9627b61c176ca70f66e43b4e94d08e" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 20828, - "sha256": "9b74155fb9bfe44ba838d64b1a31768b83978b154bdd8eaa606f494c816f189f" + "bytes": 21900, + "sha256": "d9781994f88cc95ad8562fabf24df08923bdd6d07723fddb4e0825b68c01eaac" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 23509, - "sha256": "9e4c98e43356fdf891ad35fb220abee9f9688aa58c63da6513b057cf4da3dc3f" + "bytes": 24566, + "sha256": "dc7fba17bf818b51cdbd73f365bf3c77a34fb201826f83ec8c5dd348e1125f5b" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 19319, - "sha256": "46f801051ec075725059e4e2682ef2162fd892b5e7255c8d5121d7b24dcc7f2c" + "bytes": 20060, + "sha256": "e57fba2812b36b9a4f6371bbcba79319e17ea50cee8e45f05517ca0f2a00d165" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 43290, - "sha256": "ca5509a680f94deb6583fcbb1d4e0568d2632d4ff0656b89eddc048a589b275d" + "bytes": 44354, + "sha256": "7a63cfae46d1c8482e20c8a0de57878e56821c59d980df3951e921d0a7bf5ac1" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 14879, - "sha256": "95ccf9e35708a5c290a75734632d9f76ef3a4a668d8ed6d52d2c6dabf7a85a1f" + "bytes": 15904, + "sha256": "ebc2b7522e6fdb331da08703f309a9339bc00d70f0a7346944e94325d1974351" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 26205, - "sha256": "1bba953cccaed91f75e55a9f01c0877cfb3815646ea1a7d0a0e5d6e110cca9b1" + "bytes": 27292, + "sha256": "21d389acfd5b1774ad4adb2245163bcea02bd1449b87eda24e9d522456fcd070" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 15908, - "sha256": "e876eef61d56ab7d591b57372c75ab89a60bdfd6cb5a9bdeb845025fc1781659" + "bytes": 16629, + "sha256": "5f7b0903e9575af2902b149727a096ee3d1046d20bd4d3f656068eb11d67598a" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 22070, - "sha256": "f448c426d9eae788c50a96ec88632be392c7de38199248cae28005d27a1b3fac" + "bytes": 22883, + "sha256": "075188db20b17d61de12a0df3c34fcddf1da0a26d7a6a45fbf33ed897185f225" }, { "label": "world_class_claim_guard", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 0d7de72b..bae787d4 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,18 +1,18 @@ # Benchmark Reproducibility Generated at: `2026-06-17` -Commit: `61898d145c9ca6a20a27faa28708781625757b86` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `55b50f5f8c8f4f305f9930044f132bde0b216f6556febcb5b530655b17b64702` +Commit: `d709f043a2c286646a5c72d7345faaed932200de` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `e9d5b26e20c8` +- source contract sha256: `83ea6e60852f` - archive sha256: `563d37a1085a` - output cases: `5` - disclosed failure cases: `3` @@ -20,10 +20,10 @@ Evidence bundle SHA256: `55b50f5f8c8f4f305f9930044f132bde0b216f6556febcb5b530655 - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- world-class source checks: `8` pass / `16` total; `8` blocked +- world-class source checks: `9` pass / `17` total; `8` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `32` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,22 +35,23 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | -| world-class source checks are not all accepted (8/16 pass, 8 blocked) | +| world-class source checks are not all accepted (9/17 pass, 8 blocked) | ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `55b50f5f8c8f4f305f9930044f132bde0b216f6556febcb5b530655b17b64702` +- sha256: `16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec` ## Methodology Sections @@ -72,25 +73,25 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `282122e93b08` | +| output_execution | `reports/output_execution_runs.json` | present | `ce151bc7d860` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | -| review_adjudication | `reports/output_review_adjudication.json` | present | `4843590cec50` | +| review_adjudication | `reports/output_review_adjudication.json` | present | `2ed79f8bc7a3` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `d865e5cc97a3` | -| python_compatibility | `reports/python_compatibility.json` | present | `f8e498d295db` | -| registry_audit | `reports/registry_audit.json` | present | `88c94a496b94` | +| trust_report | `reports/security_trust_report.json` | present | `f110df76d439` | +| python_compatibility | `reports/python_compatibility.json` | present | `636583616dbc` | +| registry_audit | `reports/registry_audit.json` | present | `c28991bdf087` | | package_verification | `reports/package_verification.json` | present | `f9c6cdf1a7e0` | -| install_simulation | `reports/install_simulation.json` | present | `044b2cf73dd4` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `e63bc60e69cd` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `9b74155fb9bf` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `9e4c98e43356` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `46f801051ec0` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `ca5509a680f9` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `95ccf9e35708` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `1bba953cccae` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `e876eef61d56` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `f448c426d9ea` | +| install_simulation | `reports/install_simulation.json` | present | `c91884a91538` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `4f193d15332d` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `d9781994f88c` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `dc7fba17bf81` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `e57fba2812b3` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `7a63cfae46d1` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `ebc2b7522e6f` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `21d389acfd5b` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `5f7b0903e957` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `075188db20b1` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` | ## Reproduction Commands diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 91b29bdc..39592ecb 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 496240/120000; top deferred scripts 435346; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 498644/120000; top deferred scripts 437614; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 496240/120000; top deferred scripts 435346; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 498644/120000; top deferred scripts 437614; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -222,7 +222,7 @@ "actual": { "available": true, "clean": false, - "changed_file_count": 30 + "changed_file_count": 9 }, "paths": [ "reports/benchmark_reproducibility.json" @@ -263,8 +263,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "61898d145c9ca6a20a27faa28708781625757b86", - "actual": "61898d145c9ca6a20a27faa28708781625757b86", + "expected": "d709f043a2c286646a5c72d7345faaed932200de", + "actual": "d709f043a2c286646a5c72d7345faaed932200de", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -282,8 +282,8 @@ "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 4 @@ -295,8 +295,8 @@ "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 4 @@ -355,8 +355,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -367,8 +367,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -388,16 +388,16 @@ "decision": "evidence-pending", "pending_count": 4, "accepted_count": 0, - "source_check_count": 16, - "source_pass_count": 8 + "source_check_count": 17, + "source_pass_count": 9 }, "actual": { "ready": false, "decision": "evidence-pending", "pending_count": 4, "accepted_count": 0, - "source_check_count": 16, - "source_pass_count": 8 + "source_check_count": 17, + "source_pass_count": 9 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -409,8 +409,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "61898d145c9ca6a20a27faa28708781625757b86", - "actual": "61898d145c9ca6a20a27faa28708781625757b86", + "expected": "d709f043a2c286646a5c72d7345faaed932200de", + "actual": "d709f043a2c286646a5c72d7345faaed932200de", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -428,8 +428,8 @@ "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 4 @@ -441,8 +441,8 @@ "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 4 @@ -501,8 +501,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -513,8 +513,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -534,16 +534,16 @@ "decision": "evidence-pending", "pending_count": 4, "accepted_count": 0, - "source_check_count": 16, - "source_pass_count": 8 + "source_check_count": 17, + "source_pass_count": 9 }, "actual": { "ready": false, "decision": "evidence-pending", "pending_count": 4, "accepted_count": 0, - "source_check_count": 16, - "source_pass_count": 8 + "source_check_count": 17, + "source_pass_count": 9 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -1341,8 +1341,8 @@ "accepted_count": 0, "external_pending_count": 3, "human_pending_count": 1, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1415,8 +1415,8 @@ "accepted_count": 0, "external_pending_count": 3, "human_pending_count": 1, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1825,15 +1825,15 @@ "status": "pass", "expected": { "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false }, "actual": { "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false }, @@ -1849,8 +1849,8 @@ "status": "pass", "expected": { "pending_count": 4, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, @@ -1858,8 +1858,8 @@ }, "actual": { "pending_count": 4, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, @@ -2113,8 +2113,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "plan_keys": [ "human-adjudication", @@ -2262,8 +2262,8 @@ "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "plan_keys": [ "human-adjudication", diff --git a/reports/review-studio.json b/reports/review-studio.json index 537943d6..0c0ec012 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -1985,7 +1985,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7b002075266bc2c812349c64021a8057ff5f565c47c4249a56f72651f24b7380", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, @@ -2007,9 +2007,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 49 + "changed_file_count": 32 }, - "commit": "774b77632a5b909be870d11c4ebd777c4e758539", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -2152,7 +2152,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "c5889579e024950537ee08ab56d7a63b882288515e94bbd4e66dd778f5dbef8a", + "package_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e" }, "compatibility": { @@ -2284,7 +2284,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "c5889579e024950537ee08ab56d7a63b882288515e94bbd4e66dd778f5dbef8a" + "to": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995" } ] }, @@ -4769,7 +4769,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.31, + "duration_ms": 26.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4797,7 +4797,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.16, + "duration_ms": 26.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4820,7 +4820,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.53, + "duration_ms": 27.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4848,7 +4848,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.55, + "duration_ms": 26.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4871,7 +4871,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.03, + "duration_ms": 30.27, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4899,7 +4899,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.81, + "duration_ms": 31.59, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4922,7 +4922,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.13, + "duration_ms": 30.08, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4949,7 +4949,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.71, + "duration_ms": 28.0, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4972,7 +4972,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.22, + "duration_ms": 28.31, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5001,7 +5001,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.06, + "duration_ms": 28.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5740,24 +5740,24 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "774b77632a5b909be870d11c4ebd777c4e758539", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "git_status": { "available": true, "dirty": true, - "changed_file_count": 49, + "changed_file_count": 32, "sample": [ " M registry/index.json", " M registry/packages/yao-meta-skill.json", " M reports/adoption_drift_report.json", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", " M reports/compiled_targets.json", " M reports/context_budget.json", - " M reports/context_budget.md", - " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", " M reports/install_simulation.json", - " M reports/install_simulation.md", - " M reports/output_execution_runs.json" + " M reports/install_simulation.md" ], "scope": "generation-time status before this report is written" }, @@ -5767,7 +5767,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7b002075266bc2c812349c64021a8057ff5f565c47c4249a56f72651f24b7380", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, @@ -5789,7 +5789,7 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 49 + "changed_file_count": 32 }, "public_claim": { "ready": false, @@ -5805,7 +5805,7 @@ }, "release_lock": { "ready": false, - "commit": "774b77632a5b909be870d11c4ebd777c4e758539", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "status_scope": "generation-time status before this report is written", "reason": "working tree was dirty at generation time" }, @@ -5815,7 +5815,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "7b002075266bc2c812349c64021a8057ff5f565c47c4249a56f72651f24b7380" + "sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5888,8 +5888,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "3261c0b8111c7b30c7c71bbc1ca0854f111a070865edffbaf47d2a12fa748b96" + "bytes": 7963, + "sha256": "ce151bc7d860840d0308731c3cf0342cae0610489d4205b452bbd4e47cc607ef" }, { "label": "blind_review", @@ -5938,7 +5938,7 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "e951d46b66b12547dab3c321432175285b72113221845336409bd3b5060cdbd6" + "sha256": "c28991bdf087bb31370774c1dcce9987ed41eac46b2415eecda092692b9dd915" }, { "label": "package_verification", @@ -17296,9 +17296,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1094770, + "other_text_tokens": 1094768, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1095567, + "estimated_total_text_tokens": 1095565, "deferred_resource_tokens": 498644, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ @@ -19269,7 +19269,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16T18:58:47Z", + "generated_at": "2026-06-16T19:04:13Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -24540,7 +24540,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "c5889579e024950537ee08ab56d7a63b882288515e94bbd4e66dd778f5dbef8a", + "package_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e" }, "compatibility": { @@ -24597,7 +24597,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "c5889579e024950537ee08ab56d7a63b882288515e94bbd4e66dd778f5dbef8a" + "package_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995" } ] }, @@ -25633,7 +25633,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "c5889579e024950537ee08ab56d7a63b882288515e94bbd4e66dd778f5dbef8a" + "to": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index a29e6509..a0b2368b 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -102,7 +102,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16914 个词/字。", + "入口约 360 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -206,7 +206,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16914 个词/字。" + "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -411,8 +411,8 @@ "accepted_count": 0, "external_pending_count": 3, "human_pending_count": 1, - "source_check_count": 13, - "source_pass_count": 7, + "source_check_count": 17, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -443,7 +443,8 @@ "summary_en": "Blind-review pairs still need real reviewer decisions.", "blocked_checks": [ "No pending decisions", - "Judgments complete" + "Judgments complete", + "Reviewer metadata" ] }, { @@ -982,6 +983,9 @@ "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, + "reviewer_metadata_present": false, + "reason_required": true, + "ready_for_human_evidence": false, "failure_count": 0, "reviewer_checklist_count": 5, "reviewer_checklist_pending_count": 5, @@ -996,12 +1000,12 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "ab0eb24139e6364b5ad3a386889269a178f27977569488966ce5fa7c29c2c5f8", - "source_contract_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", + "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1016,15 +1020,15 @@ "world_class_open_gap_count": 4, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 13, - "world_class_source_pass_count": 7, - "world_class_source_blocked_count": 6, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 32 }, - "commit": "4047cfdb1a8303bce568af978fb8a5462856be43", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1129,7 +1133,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 239, - "package_sha256": "e9d5b26e20c81f565271a1ecd793981ee1e56826313b160aebe776d2f89ca995" + "package_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7" }, "skill_atlas": { "skill_count": 12, @@ -1204,7 +1208,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-17" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1519,7 +1523,7 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, and required reason only.", "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -1530,6 +1534,9 @@ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], "evidence_artifacts": [ @@ -1546,7 +1553,9 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." @@ -1651,9 +1660,9 @@ "reviewer_approved_submission_count": 0, "missing_submission_count": 4, "invalid_submission_count": 0, - "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_check_count": 17, + "source_pass_count": 9, + "source_blocked_count": 8, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1777,7 +1786,7 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, and required reason only.", "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -1793,6 +1802,9 @@ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], "evidence_artifacts": [ @@ -1809,7 +1821,9 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "observed_state": { @@ -1818,6 +1832,11 @@ "pending_count": 5, "invalid_decision_count": 0, "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, "accepted": false }, "source_checklist": [ @@ -1839,7 +1858,7 @@ "actual": 5, "status": "blocked", "source_accepted": false, - "next_action": "Record a reviewer choice for every pair." + "next_action": "Record a reviewer choice and reason for every pair." }, { "evidence_key": "human-adjudication", @@ -1860,11 +1879,51 @@ "status": "pass", "source_accepted": false, "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with reviewer metadata and rationale." } ], - "source_check_count": 4, - "source_pass_count": 2, - "source_blocked_count": 2, + "source_check_count": 8, + "source_pass_count": 4, + "source_blocked_count": 4, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index 2136a136..b48e2d1d 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -929,7 +929,7 @@ -

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks8 / 16通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks9 / 17通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 329c9db4..bcc3e95c 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -411,8 +411,8 @@ "accepted_count": 0, "external_pending_count": 3, "human_pending_count": 1, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1008,8 +1008,8 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f53b2ecd4e42a3d09c7d454d210aeb647e5c2d47dc0dbe77f5e4628ff6635478", - "source_contract_sha256": "cb0a16d88e6cb8b57657714ea2ecc72ace64164de63aa1903b86b545ba20e8fc", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", + "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1024,15 +1024,15 @@ "world_class_open_gap_count": 4, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 55 + "changed_file_count": 32 }, - "commit": "f9fc7d333b5308e4530041972df14434b23e5a02", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1137,7 +1137,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 239, - "package_sha256": "cb0a16d88e6cb8b57657714ea2ecc72ace64164de63aa1903b86b545ba20e8fc" + "package_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7" }, "skill_atlas": { "skill_count": 12, @@ -1539,6 +1539,7 @@ "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], @@ -1558,6 +1559,7 @@ "Reviewer decisions should not include raw user data or private customer detail.", "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." @@ -1662,8 +1664,8 @@ "reviewer_approved_submission_count": 0, "missing_submission_count": 4, "invalid_submission_count": 0, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, @@ -1805,6 +1807,7 @@ "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], @@ -1824,6 +1827,7 @@ "Reviewer decisions should not include raw user data or private customer detail.", "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "observed_state": { @@ -1835,6 +1839,8 @@ "reviewer_metadata_present": false, "reason_required": true, "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, "accepted": false }, "source_checklist": [ @@ -1898,6 +1904,16 @@ "source_accepted": false, "next_action": "Keep reason mandatory for every imported or direct reviewer decision." }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, { "evidence_key": "human-adjudication", "label": "Human evidence ready", @@ -1909,8 +1925,8 @@ "next_action": "Complete all reviewer decisions with reviewer metadata and rationale." } ], - "source_check_count": 7, - "source_pass_count": 3, + "source_check_count": 8, + "source_pass_count": 4, "source_blocked_count": 4, "submission_state": { "status": "missing", diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 7eb87c94..b0ff5452 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -929,7 +929,7 @@ -

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks8 / 16通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks9 / 17通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 715bf397..176696a0 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -410,8 +410,8 @@ "accepted_count": 0, "external_pending_count": 3, "human_pending_count": 1, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1003,8 +1003,8 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "f53b2ecd4e42a3d09c7d454d210aeb647e5c2d47dc0dbe77f5e4628ff6635478", - "source_contract_sha256": "cb0a16d88e6cb8b57657714ea2ecc72ace64164de63aa1903b86b545ba20e8fc", + "evidence_bundle_sha256": "16f2d277d5d15e9c1c1605610f163e7060e52bf80f6d5bca4f8ccc94595ab1ec", + "source_contract_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7", "archive_sha256": "563d37a1085a9b598bca7b132da6c50c1233864855591085b260d4d4bb312a3e", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1019,15 +1019,15 @@ "world_class_open_gap_count": 4, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, - "world_class_source_check_count": 16, - "world_class_source_pass_count": 8, + "world_class_source_check_count": 17, + "world_class_source_pass_count": 9, "world_class_source_blocked_count": 8, "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 55 + "changed_file_count": 32 }, - "commit": "f9fc7d333b5308e4530041972df14434b23e5a02", + "commit": "d709f043a2c286646a5c72d7345faaed932200de", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1132,7 +1132,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 239, - "package_sha256": "cb0a16d88e6cb8b57657714ea2ecc72ace64164de63aa1903b86b545ba20e8fc" + "package_sha256": "83ea6e60852fae9271eb72658d8b3c0268e6adb21d80b2fd0fd9438805481ca7" }, "skill_atlas": { "skill_count": 12, @@ -1534,6 +1534,7 @@ "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], @@ -1553,6 +1554,7 @@ "Reviewer decisions should not include raw user data or private customer detail.", "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." @@ -1657,8 +1659,8 @@ "reviewer_approved_submission_count": 0, "missing_submission_count": 4, "invalid_submission_count": 0, - "source_check_count": 16, - "source_pass_count": 8, + "source_check_count": 17, + "source_pass_count": 9, "source_blocked_count": 8, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, @@ -1800,6 +1802,7 @@ "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", "reports/output_review_adjudication.json summary.invalid_decision_count == 0", "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", "reports/skill_os2_audit.json item human-adjudication status becomes pass" ], @@ -1819,6 +1822,7 @@ "Reviewer decisions should not include raw user data or private customer detail.", "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", "Keep the answer key separate until after decisions are recorded." ], "observed_state": { @@ -1830,6 +1834,8 @@ "reviewer_metadata_present": false, "reason_required": true, "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, "accepted": false }, "source_checklist": [ @@ -1893,6 +1899,16 @@ "source_accepted": false, "next_action": "Keep reason mandatory for every imported or direct reviewer decision." }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, { "evidence_key": "human-adjudication", "label": "Human evidence ready", @@ -1904,8 +1920,8 @@ "next_action": "Complete all reviewer decisions with reviewer metadata and rationale." } ], - "source_check_count": 7, - "source_pass_count": 3, + "source_check_count": 8, + "source_pass_count": 4, "source_blocked_count": 4, "submission_state": { "status": "missing",