From 65e906d88eb2aaf1a8ecf464bd342a8f70113842 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 00:57:50 +0800 Subject: [PATCH] Check world-class evidence workflow coverage --- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/benchmark_reproducibility.json | 54 ++-- reports/benchmark_reproducibility.md | 31 ++- reports/evidence_consistency.json | 366 +++++++++++++++++++++++-- reports/evidence_consistency.md | 7 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/package_verification.json | 2 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 12 +- reports/review-studio.json | 86 +++--- reports/review-viewer.json | 30 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-interpretation.json | 30 +- reports/skill-overview.json | 30 +- reports/upgrade_check.json | 4 +- scripts/render_evidence_consistency.py | 200 ++++++++++++++ tests/verify_evidence_consistency.py | 44 ++- 22 files changed, 770 insertions(+), 192 deletions(-) diff --git a/registry/index.json b/registry/index.json index 792ea56..f747498 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 5e5f1a0..460e6c7 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 8413704..5510679 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,23 +3,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 30, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/evidence_consistency.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c", - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 30 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "ready": false, + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c" + "sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -137,8 +151,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "0e4a474685833f04094d42f56dd84af5722185ece221fa6b63ec0996e84921d0" + "bytes": 7964, + "sha256": "30b43454ef2a4802b13d554100789b1c9c7a1baec9bebcd6fc8cf0ea9830ba41" }, { "label": "blind_review", @@ -173,7 +187,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 109099, - "sha256": "4fd98fe60dcdd1a0b8c6d0e6569bd21b5a07d695a0987a2f771d4e7b25fdec46" + "sha256": "491354ae881f6682687a62a98c9a4988bbeef931b24565fdf43e6f3a206fd449" }, { "label": "python_compatibility", @@ -187,14 +201,14 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "7c901c1abbfa4ef1e8216f5ff4426fd41f929034722914d35d5d9995c6b215e7" + "sha256": "870c455a5cd209ff8d9e8793861da074e668083d265230d04e38e8ee0ff1323a" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "0ffbc5dcb848ab2965a6e01708d7c39d243c98b8fa387ecef5280d04f112da6e" + "sha256": "c4acb824482d992e0e4339b7eeeb8d23d49da12d3414606222f4916c3ed09df6" }, { "label": "install_simulation", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index f323233..44f9dd3 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,19 +1,19 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `0979c8753e01e9cbcfbcd9905b160b8f3b6f5372` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c` +Commit: `c9a7842991900287ce259e1b624b7f36e3750265` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `24` - missing artifacts: `0` -- source contract sha256: `caa815571f7c` -- archive sha256: `6721f9af9fc9` +- source contract sha256: `f48a71e36bdb` +- archive sha256: `00151114ce7f` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `22` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `30` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `24` / `24` -- sha256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c` +- sha256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe` ## Methodology Sections @@ -72,15 +73,15 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `0e4a47468583` | +| output_execution | `reports/output_execution_runs.json` | present | `30b43454ef2a` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `4fd98fe60dcd` | +| trust_report | `reports/security_trust_report.json` | present | `491354ae881f` | | python_compatibility | `reports/python_compatibility.json` | present | `be0b8550a440` | -| registry_audit | `reports/registry_audit.json` | present | `7c901c1abbfa` | -| package_verification | `reports/package_verification.json` | present | `0ffbc5dcb848` | +| registry_audit | `reports/registry_audit.json` | present | `870c455a5cd2` | +| package_verification | `reports/package_verification.json` | present | `c4acb824482d` | | install_simulation | `reports/install_simulation.json` | present | `28ceb014e202` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `57536bc67370` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `76a3f8e2b12b` | diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index c62f46f..15ab4bc 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -4,14 +4,14 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "check_count": 28, - "pass_count": 28, + "check_count": 29, + "pass_count": 29, "warn_count": 0, "fail_count": 0, "decision": "consistent" }, "status_counts": { - "pass": 28, + "pass": 29, "warn": 0, "fail": 0 }, @@ -28,6 +28,10 @@ "reports/skill-interpretation.json", "reports/adoption_drift_report.json", "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", "reports/skill_os2_coverage.json", "reports/review-studio.json", "reports/package_verification.json", @@ -43,8 +47,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -54,8 +58,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", - "actual": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "expected": "c9a7842991900287ce259e1b624b7f36e3750265", + "actual": "c9a7842991900287ce259e1b624b7f36e3750265", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -67,30 +71,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -200,8 +204,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", - "actual": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "expected": "c9a7842991900287ce259e1b624b7f36e3750265", + "actual": "c9a7842991900287ce259e1b624b7f36e3750265", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -213,30 +217,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -1704,6 +1708,318 @@ ], "detail": "The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted." }, + { + "key": "world-class-evidence-workflow-coverage", + "label": "World-class evidence workflows cover every pending ledger entry", + "status": "pass", + "expected": { + "keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "plan_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "submission_review_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_runbook_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "review_studio_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_ready_to_claim_world_class": false, + "submission_review_ready_to_claim_world_class": false, + "submission_review_counts_as_completion": false, + "operator_runbook_ready_to_claim_world_class": false, + "operator_runbook_counts_as_completion": false, + "next_actions_present": { + "human-adjudication": true, + "native-client-telemetry": true, + "native-permission-enforcement": true, + "provider-holdout": true + }, + "commands": { + "human-adjudication": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-client-telemetry": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-permission-enforcement": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "provider-holdout": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + } + } + }, + "actual": { + "keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "plan_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "submission_review_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_runbook_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "review_studio_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_ready_to_claim_world_class": false, + "submission_review_ready_to_claim_world_class": false, + "submission_review_counts_as_completion": false, + "operator_runbook_ready_to_claim_world_class": false, + "operator_runbook_counts_as_completion": false, + "next_actions_present": { + "human-adjudication": true, + "native-client-telemetry": true, + "native-permission-enforcement": true, + "provider-holdout": true + }, + "commands": { + "human-adjudication": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-client-telemetry": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-permission-enforcement": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "provider-holdout": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + } + } + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", + "reports/review-studio.json" + ], + "detail": "Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion." + }, { "key": "skill-os-2-review-current-evidence", "label": "Skill OS 2.0 review summary mirrors current evidence", diff --git a/reports/evidence_consistency.md b/reports/evidence_consistency.md index 1b8596a..6245d0e 100644 --- a/reports/evidence_consistency.md +++ b/reports/evidence_consistency.md @@ -5,8 +5,8 @@ Generated at: `2026-06-16` ## Summary - decision: `consistent` -- checks: `28` -- pass: `28` +- checks: `29` +- pass: `29` - warn: `0` - fail: `0` @@ -16,7 +16,7 @@ This gate compares generated evidence reports against each other. It does not cr | Check | Status | Detail | Paths | | --- | --- | --- | --- | -| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/world_class_claim_guard.json`, `reports/skill-os-2-review.md` | +| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/world_class_claim_guard.json`, `reports/skill-os-2-review.md` | | Benchmark release lock matches git dirty state | `pass` | The benchmark release lock must reflect the generation-time git dirty flag. | `reports/benchmark_reproducibility.json` | | overview embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | | overview embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | @@ -43,4 +43,5 @@ This gate compares generated evidence reports against each other. It does not cr | Benchmark report mirrors world-class evidence boundary | `pass` | Benchmark reproducibility must not overstate public claim readiness. | `reports/world_class_evidence_ledger.json`, `reports/benchmark_reproducibility.json` | | Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` | | Claim guard covers package and runtime claim surfaces | `pass` | The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted. | `reports/world_class_claim_guard.json`, `manifest.json`, `agents/interface.yaml`, `dist/manifest.json`, `dist/targets/openai/adapter.json`, `evidence/world_class/README.md`, `security/permission_policy.json`, `reports/world_class_evidence_ledger.json` | +| World-class evidence workflows cover every pending ledger entry | `pass` | Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` | | Skill OS 2.0 review summary mirrors current evidence | `pass` | Manual 2.0 review summaries must not drift from generated gate, package, trust, context, benchmark, or CI evidence. | `reports/skill-os-2-review.md`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/benchmark_reproducibility.json`, `scripts/ci_test.py` | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 57cb218..5210d93 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.46, + "duration_ms": 26.28, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.34, + "duration_ms": 26.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.22, + "duration_ms": 26.08, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.54, + "duration_ms": 26.33, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.72, + "duration_ms": 25.89, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.47, + "duration_ms": 26.56, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.05, + "duration_ms": 26.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.35, + "duration_ms": 26.4, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.32, + "duration_ms": 26.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.65, + "duration_ms": 26.1, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 6772476..ea98986 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 26.46 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.34 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.22 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.54 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 26.72 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.47 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.05 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.35 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.32 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.65 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.28 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.19 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 26.08 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.33 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 25.89 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.56 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.12 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.4 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.2 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.1 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 01c0bea..b600a68 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index fa31c47..df06b5e 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f` +- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 76454a2..b0faf6c 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 5c8adc6..e49d750 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921` -- Archive SHA256: `6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f` +- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086` +- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 954499f..0a0fc48 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -676,7 +676,7 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

110 scripts scanned; secrets found

Py Compat0

176 files scanned for Python 3.11

Arch Debt0

899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

617 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

110 scripts scanned; secrets found

Py Compat0

176 files scanned for Python 3.11

Arch Debt0

899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

617 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

@@ -748,7 +748,7 @@
-

信任报告

Secret
0
脚本数
110
网络脚本
3
Help 失败
0
包体哈希
caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921
+

信任报告

Secret
0
脚本数
110
网络脚本
3
Help 失败
0
包体哈希
f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -806,8 +806,8 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
+

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
@@ -827,12 +827,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
617
失败数
0
警告数
0
归档哈希
6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
617
失败数
0
警告数
0
归档哈希
00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137
diff --git a/reports/review-studio.json b/reports/review-studio.json index 0128716..595e0ea 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -1676,7 +1676,7 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, @@ -1700,11 +1700,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 2 }, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1809,7 +1809,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "skill_atlas": { "skill_count": 12, @@ -1847,8 +1847,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -1879,7 +1879,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1895,7 +1895,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 @@ -1974,12 +1974,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, @@ -4346,7 +4346,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.46, + "duration_ms": 26.63, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4374,7 +4374,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.34, + "duration_ms": 26.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4397,7 +4397,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.22, + "duration_ms": 26.15, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4425,7 +4425,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.54, + "duration_ms": 29.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4448,7 +4448,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.72, + "duration_ms": 29.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4476,7 +4476,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.47, + "duration_ms": 27.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4499,7 +4499,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.05, + "duration_ms": 25.82, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4526,7 +4526,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.35, + "duration_ms": 26.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4549,7 +4549,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.32, + "duration_ms": 26.09, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4578,7 +4578,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.65, + "duration_ms": 26.0, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5299,17 +5299,20 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 2, + "sample": [ + " M scripts/render_evidence_consistency.py", + " M tests/verify_evidence_consistency.py" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, @@ -5333,14 +5336,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 2 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5349,10 +5353,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "ready": false, + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -11412,7 +11416,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "failures": [], "warnings": [], @@ -20438,8 +20442,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -20470,7 +20474,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -20495,7 +20499,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, @@ -20518,7 +20522,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 @@ -21526,12 +21530,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 9a514a6..8161618 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -996,13 +996,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c", - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 30 }, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1129,7 +1129,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "skill_atlas": { "skill_count": 12, @@ -1167,8 +1167,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -1199,7 +1199,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1215,7 +1215,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 @@ -1294,12 +1294,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 3052a64..95738ce 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index bb00c2f..91164b7 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `197` -- Package SHA256: `caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921` +- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086` ## Failures diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 85ec652..ec3303f 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1000,13 +1000,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c", - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1024,11 +1024,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 30 }, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1133,7 +1133,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "skill_atlas": { "skill_count": 12, @@ -1171,8 +1171,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -1203,7 +1203,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1219,7 +1219,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 @@ -1298,12 +1298,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/reports/skill-overview.json b/reports/skill-overview.json index ebc4e06..77babfa 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -995,13 +995,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c", - "source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1019,11 +1019,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 30 }, - "commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372", + "commit": "c9a7842991900287ce259e1b624b7f36e3750265", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1128,7 +1128,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 197, - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" }, "skill_atlas": { "skill_count": 12, @@ -1166,8 +1166,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921", - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, "compatibility": { "openai": "pass", @@ -1198,7 +1198,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1214,7 +1214,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f", + "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "archive_entry_count": 617, "failure_count": 0, "warning_count": 0 @@ -1293,12 +1293,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index e257998..ea4a7ff 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f" + "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921" + "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" } ] }, diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index d6d3394..2f24a49 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -16,6 +16,10 @@ REQUIRED_REPORTS = { "interpretation": "reports/skill-interpretation.json", "adoption": "reports/adoption_drift_report.json", "world_class_ledger": "reports/world_class_evidence_ledger.json", + "world_class_plan": "reports/world_class_evidence_plan.json", + "world_class_intake": "reports/world_class_evidence_intake.json", + "world_class_submission_review": "reports/world_class_submission_review.json", + "world_class_operator_runbook": "reports/world_class_operator_runbook.json", "skill_os2_coverage": "reports/skill_os2_coverage.json", "review_studio": "reports/review-studio.json", "package_verification": "reports/package_verification.json", @@ -145,6 +149,69 @@ def as_int(value: Any) -> int | None: return None +def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]: + collection = payload.get(collection_key) + if not isinstance(collection, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for item in collection: + if not isinstance(item, dict): + continue + key = item.get("key") or item.get("evidence_key") + if isinstance(key, str): + keyed[key] = item + return keyed + + +def command_key_set(item: dict[str, Any]) -> set[str]: + commands = item.get("commands") + if isinstance(commands, dict): + return {key for key, value in commands.items() if isinstance(key, str) and value} + if isinstance(commands, list): + keys: set[str] = set() + for command in commands: + if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"): + keys.add(command["key"]) + return keys + return set() + + +def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: + actions = review_studio.get("review_actions") + if not isinstance(actions, list): + return {} + for action in actions: + if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": + continue + steps = action.get("evidence_steps") + if not isinstance(steps, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for step in steps: + if isinstance(step, dict) and isinstance(step.get("key"), str): + keyed[step["key"]] = step + return keyed + return {} + + +def command_groups_present(command_keys: set[str]) -> dict[str, bool]: + return { + "prepare_submission": "prepare_submission" in command_keys, + "validate_intake": "validate_intake" in command_keys, + "submission_review": bool({"submission_review", "review_queue"} & command_keys), + "refresh_ledger": "refresh_ledger" in command_keys, + "guard_claim": "guard_claim" in command_keys, + } + + +def has_next_action(item: dict[str, Any]) -> bool: + for key in ["next_action", "audit_next_action"]: + value = item.get(key) + if isinstance(value, str) and value.strip(): + return True + return False + + def add_check( checks: list[dict[str, Any]], *, @@ -250,6 +317,10 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: interpretation = reports["interpretation"] adoption = reports["adoption"] ledger = reports["world_class_ledger"] + world_class_plan = reports["world_class_plan"] + world_class_intake = reports["world_class_intake"] + world_class_submission_review = reports["world_class_submission_review"] + world_class_operator_runbook = reports["world_class_operator_runbook"] coverage = reports["skill_os2_coverage"] review_studio = reports["review_studio"] package_verification = reports["package_verification"] @@ -261,6 +332,10 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: benchmark_summary = nested(benchmark, ["summary"], {}) adoption_summary = nested(adoption, ["summary"], {}) ledger_summary = nested(ledger, ["summary"], {}) + plan_summary = nested(world_class_plan, ["summary"], {}) + intake_summary = nested(world_class_intake, ["summary"], {}) + submission_review_summary = nested(world_class_submission_review, ["summary"], {}) + operator_runbook_summary = nested(world_class_operator_runbook, ["summary"], {}) coverage_summary = nested(coverage, ["summary"], {}) studio_summary = nested(review_studio, ["summary"], {}) package_summary = nested(package_verification, ["summary"], {}) @@ -510,6 +585,131 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: ], detail="The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted.", ) + ledger_items = keyed_items(ledger, "entries") + plan_tasks = keyed_items(world_class_plan, "tasks") + intake_checklist = keyed_items(world_class_intake, "operator_checklist") + submission_review_items = keyed_items(world_class_submission_review, "items") + operator_runbook_items = keyed_items(world_class_operator_runbook, "items") + review_action_steps = world_class_review_action_steps(review_studio) + pending_world_class_keys = sorted( + key for key, item in ledger_items.items() if item.get("status") != "accepted" + ) + expected_command_groups = { + key: { + "intake": { + "prepare_submission": True, + "validate_intake": True, + "submission_review": True, + "refresh_ledger": True, + "guard_claim": True, + }, + "operator_runbook": { + "prepare_submission": True, + "validate_intake": True, + "submission_review": True, + "refresh_ledger": True, + "guard_claim": True, + }, + "review_studio": { + "prepare_submission": True, + "validate_intake": True, + "submission_review": True, + "refresh_ledger": True, + "guard_claim": True, + }, + } + for key in pending_world_class_keys + } + actual_command_groups = { + key: { + "intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))), + "operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))), + "review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))), + } + for key in pending_world_class_keys + } + expected_world_class_workflow = { + "keys": pending_world_class_keys, + "pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, + "human_pending_count": ledger_summary.get("human_pending_count") if isinstance(ledger_summary, dict) else None, + "external_pending_count": ledger_summary.get("external_pending_count") if isinstance(ledger_summary, dict) else None, + "source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None, + "source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None, + "source_blocked_count": ledger_summary.get("source_blocked_count") if isinstance(ledger_summary, dict) else None, + "plan_keys": pending_world_class_keys, + "intake_keys": pending_world_class_keys, + "submission_review_keys": pending_world_class_keys, + "operator_runbook_keys": pending_world_class_keys, + "review_studio_keys": pending_world_class_keys, + "intake_ready_to_claim_world_class": False, + "submission_review_ready_to_claim_world_class": False, + "submission_review_counts_as_completion": False, + "operator_runbook_ready_to_claim_world_class": False, + "operator_runbook_counts_as_completion": False, + "next_actions_present": {key: True for key in pending_world_class_keys}, + "commands": expected_command_groups, + } + actual_world_class_workflow = { + "keys": pending_world_class_keys, + "pending_count": plan_summary.get("task_count") if isinstance(plan_summary, dict) else None, + "human_pending_count": plan_summary.get("human_task_count") if isinstance(plan_summary, dict) else None, + "external_pending_count": plan_summary.get("external_task_count") if isinstance(plan_summary, dict) else None, + "source_check_count": submission_review_summary.get("source_check_count") + if isinstance(submission_review_summary, dict) + else None, + "source_pass_count": submission_review_summary.get("source_pass_count") + if isinstance(submission_review_summary, dict) + else None, + "source_blocked_count": submission_review_summary.get("source_blocked_count") + if isinstance(submission_review_summary, dict) + else None, + "plan_keys": sorted(plan_tasks), + "intake_keys": sorted(intake_checklist), + "submission_review_keys": sorted(submission_review_items), + "operator_runbook_keys": sorted(operator_runbook_items), + "review_studio_keys": sorted(review_action_steps), + "intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class") + if isinstance(intake_summary, dict) + else None, + "submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class") + if isinstance(submission_review_summary, dict) + else None, + "submission_review_counts_as_completion": submission_review_summary.get( + "review_counts_submission_as_completion" + ) + if isinstance(submission_review_summary, dict) + else None, + "operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class") + if isinstance(operator_runbook_summary, dict) + else None, + "operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion") + if isinstance(operator_runbook_summary, dict) + else None, + "next_actions_present": { + key: all( + has_next_action(collection.get(key, {})) + for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps] + ) + for key in pending_world_class_keys + }, + "commands": actual_command_groups, + } + compare_values( + checks, + key="world-class-evidence-workflow-coverage", + label="World-class evidence workflows cover every pending ledger entry", + expected=expected_world_class_workflow, + actual=actual_world_class_workflow, + paths=[ + REQUIRED_REPORTS["world_class_ledger"], + REQUIRED_REPORTS["world_class_plan"], + REQUIRED_REPORTS["world_class_intake"], + REQUIRED_REPORTS["world_class_submission_review"], + REQUIRED_REPORTS["world_class_operator_runbook"], + REQUIRED_REPORTS["review_studio"], + ], + detail="Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion.", + ) skill_os2_review = text_reports.get("skill_os2_review", "") ci_target_count = ci_default_target_count(skill_dir / "scripts" / "ci_test.py") expected_review_snippets = [ diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index e648678..724bd05 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -17,6 +17,10 @@ REPORT_FILES = [ "reports/skill-interpretation.html", "reports/adoption_drift_report.json", "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", "reports/skill_os2_coverage.json", "reports/review-studio.json", "reports/package_verification.json", @@ -45,6 +49,11 @@ def refresh_embedded_reports() -> None: script_names = [ "render_benchmark_reproducibility.py", "render_skill_os2_coverage.py", + "render_world_class_evidence_plan.py", + "render_world_class_evidence_ledger.py", + "render_world_class_evidence_intake.py", + "render_world_class_submission_review.py", + "render_world_class_operator_runbook.py", "render_world_class_claim_guard.py", "render_skill_overview.py", "render_skill_interpretation.py", @@ -112,7 +121,7 @@ def main() -> None: assert payload["ok"] is True, payload assert payload["summary"]["decision"] == "consistent", payload assert payload["summary"]["fail_count"] == 0, payload - assert payload["summary"]["check_count"] >= 28, payload + assert payload["summary"]["check_count"] >= 29, payload checks = {item["key"]: item for item in payload["checks"]} assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] @@ -121,6 +130,9 @@ def main() -> None: assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[ "claim-guard-package-runtime-surface" ] + assert checks["world-class-evidence-workflow-coverage"]["status"] == "pass", checks[ + "world-class-evidence-workflow-coverage" + ] assert checks["skill-os-2-review-current-evidence"]["status"] == "pass", checks[ "skill-os-2-review-current-evidence" ] @@ -184,6 +196,36 @@ def main() -> None: assert claim_guard_drift_checks["claim-guard-package-runtime-surface"]["status"] == "fail", ( claim_guard_drift_checks["claim-guard-package-runtime-surface"] ) + + workflow_drift_root = TMP / "workflow-drift-skill" + copy_reports(workflow_drift_root) + studio_path = workflow_drift_root / "reports" / "review-studio.json" + studio = json.loads(studio_path.read_text(encoding="utf-8")) + for action in studio["review_actions"]: + if action.get("gate_key") == "world-class-evidence": + action["evidence_steps"] = [ + item for item in action["evidence_steps"] if item.get("key") != "native-client-telemetry" + ] + studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + workflow_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(workflow_drift_root), + "--output-json", + str(TMP / "workflow_drift.json"), + "--output-md", + str(TMP / "workflow_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert workflow_drift_proc.returncode == 2, workflow_drift_proc.stdout + workflow_drift_payload = json.loads(workflow_drift_proc.stdout) + workflow_drift_checks = {item["key"]: item for item in workflow_drift_payload["checks"]} + assert workflow_drift_checks["world-class-evidence-workflow-coverage"]["status"] == "fail", ( + workflow_drift_checks["world-class-evidence-workflow-coverage"] + ) print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))