Check world-class evidence workflow coverage
This commit is contained in:
yaojingang
+1
-1
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -48,7 +48,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
|
||||
@@ -3,23 +3,36 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"skill_dir": ".",
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"sample": [],
|
||||
"dirty": true,
|
||||
"changed_file_count": 30,
|
||||
"sample": [
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/context_budget.md",
|
||||
" M reports/context_budget_summary.json",
|
||||
" M reports/evidence_consistency.json",
|
||||
" M reports/evidence_consistency.md"
|
||||
],
|
||||
"scope": "generation-time status before this report is written"
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c",
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 22,
|
||||
@@ -37,14 +50,15 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 30
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": false,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": [
|
||||
"release lock is not clean or commit is unavailable",
|
||||
"provider-backed model holdout evidence is incomplete",
|
||||
"human blind-review adjudication is incomplete",
|
||||
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
|
||||
@@ -53,10 +67,10 @@
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"ready": false,
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"reason": "clean generation-time HEAD"
|
||||
"reason": "working tree was dirty at generation time"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
@@ -64,7 +78,7 @@
|
||||
"existing_count": 24,
|
||||
"missing_count": 0,
|
||||
"missing_paths": [],
|
||||
"sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c"
|
||||
"sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe"
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -137,8 +151,8 @@
|
||||
"label": "output_execution",
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "0e4a474685833f04094d42f56dd84af5722185ece221fa6b63ec0996e84921d0"
|
||||
"bytes": 7964,
|
||||
"sha256": "30b43454ef2a4802b13d554100789b1c9c7a1baec9bebcd6fc8cf0ea9830ba41"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -173,7 +187,7 @@
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 109099,
|
||||
"sha256": "4fd98fe60dcdd1a0b8c6d0e6569bd21b5a07d695a0987a2f771d4e7b25fdec46"
|
||||
"sha256": "491354ae881f6682687a62a98c9a4988bbeef931b24565fdf43e6f3a206fd449"
|
||||
},
|
||||
{
|
||||
"label": "python_compatibility",
|
||||
@@ -187,14 +201,14 @@
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "7c901c1abbfa4ef1e8216f5ff4426fd41f929034722914d35d5d9995c6b215e7"
|
||||
"sha256": "870c455a5cd209ff8d9e8793861da074e668083d265230d04e38e8ee0ff1323a"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
"path": "reports/package_verification.json",
|
||||
"exists": true,
|
||||
"bytes": 19338,
|
||||
"sha256": "0ffbc5dcb848ab2965a6e01708d7c39d243c98b8fa387ecef5280d04f112da6e"
|
||||
"sha256": "c4acb824482d992e0e4339b7eeeb8d23d49da12d3414606222f4916c3ed09df6"
|
||||
},
|
||||
{
|
||||
"label": "install_simulation",
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-16`
|
||||
Commit: `0979c8753e01e9cbcfbcd9905b160b8f3b6f5372`
|
||||
Working tree dirty at generation: `false`
|
||||
Evidence bundle SHA256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c`
|
||||
Commit: `c9a7842991900287ce259e1b624b7f36e3750265`
|
||||
Working tree dirty at generation: `true`
|
||||
Evidence bundle SHA256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- release lock ready: `true`
|
||||
- release lock ready: `false`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `24`
|
||||
- missing artifacts: `0`
|
||||
- source contract sha256: `caa815571f7c`
|
||||
- archive sha256: `6721f9af9fc9`
|
||||
- source contract sha256: `f48a71e36bdb`
|
||||
- archive sha256: `00151114ce7f`
|
||||
- output cases: `5`
|
||||
- disclosed failure cases: `3`
|
||||
- reproduction commands: `22`
|
||||
@@ -22,8 +22,8 @@ Evidence bundle SHA256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3
|
||||
- world-class ready: `false`
|
||||
- world-class source checks: `6` pass / `13` total; `7` blocked
|
||||
- public claim ready: `false`
|
||||
- public claim blockers: `4`
|
||||
- changed files at generation: `0`
|
||||
- public claim blockers: `5`
|
||||
- changed files at generation: `30`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
|
||||
|
||||
@@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
| Blocker |
|
||||
| --- |
|
||||
| release lock is not clean or commit is unavailable |
|
||||
| provider-backed model holdout evidence is incomplete |
|
||||
| human blind-review adjudication is incomplete |
|
||||
| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) |
|
||||
@@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
## Release Lock
|
||||
|
||||
- ready: `true`
|
||||
- reason: clean generation-time HEAD
|
||||
- ready: `false`
|
||||
- reason: working tree was dirty at generation time
|
||||
- status scope: generation-time status before this report is written
|
||||
|
||||
## Evidence Bundle
|
||||
|
||||
- algorithm: `sha256(path,label,exists,artifact_sha256)`
|
||||
- artifacts: `24` / `24`
|
||||
- sha256: `a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c`
|
||||
- sha256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe`
|
||||
|
||||
## Methodology Sections
|
||||
|
||||
@@ -72,15 +73,15 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `0e4a47468583` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `30b43454ef2a` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `4fd98fe60dcd` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `491354ae881f` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `be0b8550a440` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `7c901c1abbfa` |
|
||||
| package_verification | `reports/package_verification.json` | present | `0ffbc5dcb848` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `870c455a5cd2` |
|
||||
| package_verification | `reports/package_verification.json` | present | `c4acb824482d` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `28ceb014e202` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `57536bc67370` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `76a3f8e2b12b` |
|
||||
|
||||
@@ -4,14 +4,14 @@
|
||||
"generated_at": "2026-06-16",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"check_count": 28,
|
||||
"pass_count": 28,
|
||||
"check_count": 29,
|
||||
"pass_count": 29,
|
||||
"warn_count": 0,
|
||||
"fail_count": 0,
|
||||
"decision": "consistent"
|
||||
},
|
||||
"status_counts": {
|
||||
"pass": 28,
|
||||
"pass": 29,
|
||||
"warn": 0,
|
||||
"fail": 0
|
||||
},
|
||||
@@ -28,6 +28,10 @@
|
||||
"reports/skill-interpretation.json",
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
"reports/world_class_evidence_plan.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_submission_review.json",
|
||||
"reports/world_class_operator_runbook.json",
|
||||
"reports/skill_os2_coverage.json",
|
||||
"reports/review-studio.json",
|
||||
"reports/package_verification.json",
|
||||
@@ -43,8 +47,8 @@
|
||||
"key": "benchmark-release-lock-self-consistency",
|
||||
"label": "Benchmark release lock matches git dirty state",
|
||||
"status": "pass",
|
||||
"expected": true,
|
||||
"actual": true,
|
||||
"expected": false,
|
||||
"actual": false,
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json"
|
||||
],
|
||||
@@ -54,8 +58,8 @@
|
||||
"key": "overview-benchmark-commit",
|
||||
"label": "overview embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"actual": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"expected": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"actual": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-overview.json"
|
||||
@@ -67,30 +71,30 @@
|
||||
"label": "overview embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -200,8 +204,8 @@
|
||||
"key": "interpretation-benchmark-commit",
|
||||
"label": "interpretation embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"actual": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"expected": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"actual": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-interpretation.json"
|
||||
@@ -213,30 +217,30 @@
|
||||
"label": "interpretation embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -1704,6 +1708,318 @@
|
||||
],
|
||||
"detail": "The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted."
|
||||
},
|
||||
{
|
||||
"key": "world-class-evidence-workflow-coverage",
|
||||
"label": "World-class evidence workflows cover every pending ledger entry",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"plan_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"intake_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"submission_review_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"operator_runbook_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"review_studio_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"intake_ready_to_claim_world_class": false,
|
||||
"submission_review_ready_to_claim_world_class": false,
|
||||
"submission_review_counts_as_completion": false,
|
||||
"operator_runbook_ready_to_claim_world_class": false,
|
||||
"operator_runbook_counts_as_completion": false,
|
||||
"next_actions_present": {
|
||||
"human-adjudication": true,
|
||||
"native-client-telemetry": true,
|
||||
"native-permission-enforcement": true,
|
||||
"provider-holdout": true
|
||||
},
|
||||
"commands": {
|
||||
"human-adjudication": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"provider-holdout": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"actual": {
|
||||
"keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"plan_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"intake_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"submission_review_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"operator_runbook_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"review_studio_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"intake_ready_to_claim_world_class": false,
|
||||
"submission_review_ready_to_claim_world_class": false,
|
||||
"submission_review_counts_as_completion": false,
|
||||
"operator_runbook_ready_to_claim_world_class": false,
|
||||
"operator_runbook_counts_as_completion": false,
|
||||
"next_actions_present": {
|
||||
"human-adjudication": true,
|
||||
"native-client-telemetry": true,
|
||||
"native-permission-enforcement": true,
|
||||
"provider-holdout": true
|
||||
},
|
||||
"commands": {
|
||||
"human-adjudication": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
},
|
||||
"provider-holdout": {
|
||||
"intake": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": true,
|
||||
"validate_intake": true,
|
||||
"submission_review": true,
|
||||
"refresh_ledger": true,
|
||||
"guard_claim": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"paths": [
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
"reports/world_class_evidence_plan.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_submission_review.json",
|
||||
"reports/world_class_operator_runbook.json",
|
||||
"reports/review-studio.json"
|
||||
],
|
||||
"detail": "Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion."
|
||||
},
|
||||
{
|
||||
"key": "skill-os-2-review-current-evidence",
|
||||
"label": "Skill OS 2.0 review summary mirrors current evidence",
|
||||
|
||||
@@ -5,8 +5,8 @@ Generated at: `2026-06-16`
|
||||
## Summary
|
||||
|
||||
- decision: `consistent`
|
||||
- checks: `28`
|
||||
- pass: `28`
|
||||
- checks: `29`
|
||||
- pass: `29`
|
||||
- warn: `0`
|
||||
- fail: `0`
|
||||
|
||||
@@ -16,7 +16,7 @@ This gate compares generated evidence reports against each other. It does not cr
|
||||
|
||||
| Check | Status | Detail | Paths |
|
||||
| --- | --- | --- | --- |
|
||||
| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/world_class_claim_guard.json`, `reports/skill-os-2-review.md` |
|
||||
| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/world_class_claim_guard.json`, `reports/skill-os-2-review.md` |
|
||||
| Benchmark release lock matches git dirty state | `pass` | The benchmark release lock must reflect the generation-time git dirty flag. | `reports/benchmark_reproducibility.json` |
|
||||
| overview embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` |
|
||||
| overview embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` |
|
||||
@@ -43,4 +43,5 @@ This gate compares generated evidence reports against each other. It does not cr
|
||||
| Benchmark report mirrors world-class evidence boundary | `pass` | Benchmark reproducibility must not overstate public claim readiness. | `reports/world_class_evidence_ledger.json`, `reports/benchmark_reproducibility.json` |
|
||||
| Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` |
|
||||
| Claim guard covers package and runtime claim surfaces | `pass` | The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted. | `reports/world_class_claim_guard.json`, `manifest.json`, `agents/interface.yaml`, `dist/manifest.json`, `dist/targets/openai/adapter.json`, `evidence/world_class/README.md`, `security/permission_policy.json`, `reports/world_class_evidence_ledger.json` |
|
||||
| World-class evidence workflows cover every pending ledger entry | `pass` | Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` |
|
||||
| Skill OS 2.0 review summary mirrors current evidence | `pass` | Manual 2.0 review summaries must not drift from generated gate, package, trust, context, benchmark, or CI evidence. | `reports/skill-os-2-review.md`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/benchmark_reproducibility.json`, `scripts/ci_test.py` |
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.46,
|
||||
"duration_ms": 26.28,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.34,
|
||||
"duration_ms": 26.19,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.22,
|
||||
"duration_ms": 26.08,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.54,
|
||||
"duration_ms": 26.33,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.72,
|
||||
"duration_ms": 25.89,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.47,
|
||||
"duration_ms": 26.56,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.05,
|
||||
"duration_ms": 26.12,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.35,
|
||||
"duration_ms": 26.4,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.32,
|
||||
"duration_ms": 26.2,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.65,
|
||||
"duration_ms": 26.1,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 26.46 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.34 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 26.22 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.54 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 26.72 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.47 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.05 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.35 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.32 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.65 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 26.28 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.19 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 26.08 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.33 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 25.89 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.56 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.12 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.4 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.2 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.1 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
- Package directory: `dist`
|
||||
- Targets: `4 / 4` adapters present
|
||||
- Archive present: `True`
|
||||
- Archive SHA256: `6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f`
|
||||
- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137`
|
||||
- Failures: `0`
|
||||
- Warnings: `0`
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -53,7 +53,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921`
|
||||
- Archive SHA256: `6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f`
|
||||
- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086`
|
||||
- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137`
|
||||
- Install simulated: `True`
|
||||
|
||||
## Compatibility
|
||||
|
||||
@@ -676,7 +676,7 @@
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Public Claim</span><strong>blocked</strong><p>4 blockers; local reproducible true</p></article><article class='metric'><span>Blueprint</span><strong>21/21</strong><p>2.0 coverage; extensions partial 0, planned 0; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>110 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>176 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>173 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>617 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Public Claim</span><strong>blocked</strong><p>5 blockers; local reproducible true</p></article><article class='metric'><span>Blueprint</span><strong>21/21</strong><p>2.0 coverage; extensions partial 0, planned 0; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>110 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>176 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>173 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>617 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
@@ -748,7 +748,7 @@
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>110</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>110</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
@@ -806,8 +806,8 @@
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>公开声明</h2><dl class='kv-grid'><div><dt>本地复现</dt><dd>是</dd></div><div><dt>发布锁</dt><dd>是</dd></div><div><dt>可公开声明</dt><dd>否</dd></div><div><dt>声明阻断</dt><dd>4</dd></div><div><dt>Provider 证据</dt><dd>否</dd></div><div><dt>人审完成</dt><dd>否</dd></div><div><dt>世界级就绪</dt><dd>否</dd></div></dl></div>
|
||||
<div class="panel"><h2>声明阻断</h2><ul class='issues'><li><strong>阻断</strong><span>provider-backed model holdout evidence is incomplete</span></li><li><strong>阻断</strong><span>human blind-review adjudication is incomplete</span></li><li><strong>阻断</strong><span>world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)</span></li><li><strong>阻断</strong><span>world-class source checks are not all accepted (6/13 pass, 7 blocked)</span></li></ul></div>
|
||||
<div class="panel"><h2>公开声明</h2><dl class='kv-grid'><div><dt>本地复现</dt><dd>是</dd></div><div><dt>发布锁</dt><dd>否</dd></div><div><dt>可公开声明</dt><dd>否</dd></div><div><dt>声明阻断</dt><dd>5</dd></div><div><dt>Provider 证据</dt><dd>否</dd></div><div><dt>人审完成</dt><dd>否</dd></div><div><dt>世界级就绪</dt><dd>否</dd></div></dl></div>
|
||||
<div class="panel"><h2>声明阻断</h2><ul class='issues'><li><strong>阻断</strong><span>release lock is not clean or commit is unavailable</span></li><li><strong>阻断</strong><span>provider-backed model holdout evidence is incomplete</span></li><li><strong>阻断</strong><span>human blind-review adjudication is incomplete</span></li><li><strong>阻断</strong><span>world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)</span></li><li><strong>阻断</strong><span>world-class source checks are not all accepted (6/13 pass, 7 blocked)</span></li></ul></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
@@ -827,12 +827,12 @@
|
||||
|
||||
<section id="registry" class="twocol">
|
||||
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>617</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>617</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
|
||||
+45
-41
@@ -1676,7 +1676,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
@@ -1700,11 +1700,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 2
|
||||
},
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1809,7 +1809,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1847,8 +1847,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1879,7 +1879,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1895,7 +1895,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
@@ -1974,12 +1974,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -4346,7 +4346,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.46,
|
||||
"duration_ms": 26.63,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4374,7 +4374,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.34,
|
||||
"duration_ms": 26.13,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4397,7 +4397,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.22,
|
||||
"duration_ms": 26.15,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4425,7 +4425,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.54,
|
||||
"duration_ms": 29.26,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4448,7 +4448,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.72,
|
||||
"duration_ms": 29.51,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4476,7 +4476,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.47,
|
||||
"duration_ms": 27.2,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4499,7 +4499,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.05,
|
||||
"duration_ms": 25.82,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4526,7 +4526,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.35,
|
||||
"duration_ms": 26.22,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4549,7 +4549,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.32,
|
||||
"duration_ms": 26.09,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4578,7 +4578,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.65,
|
||||
"duration_ms": 26.0,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -5299,17 +5299,20 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"skill_dir": ".",
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"sample": [],
|
||||
"dirty": true,
|
||||
"changed_file_count": 2,
|
||||
"sample": [
|
||||
" M scripts/render_evidence_consistency.py",
|
||||
" M tests/verify_evidence_consistency.py"
|
||||
],
|
||||
"scope": "generation-time status before this report is written"
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
@@ -5333,14 +5336,15 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 2
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": false,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": [
|
||||
"release lock is not clean or commit is unavailable",
|
||||
"provider-backed model holdout evidence is incomplete",
|
||||
"human blind-review adjudication is incomplete",
|
||||
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
|
||||
@@ -5349,10 +5353,10 @@
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"ready": false,
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"reason": "clean generation-time HEAD"
|
||||
"reason": "working tree was dirty at generation time"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
@@ -11412,7 +11416,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -20438,8 +20442,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -20470,7 +20474,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -20495,7 +20499,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -20518,7 +20522,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
@@ -21526,12 +21530,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
+15
-15
@@ -996,13 +996,13 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c",
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 22,
|
||||
@@ -1020,11 +1020,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 30
|
||||
},
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1129,7 +1129,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1167,8 +1167,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1199,7 +1199,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1215,7 +1215,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
@@ -1294,12 +1294,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `197`
|
||||
- Package SHA256: `caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921`
|
||||
- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086`
|
||||
|
||||
## Failures
|
||||
|
||||
|
||||
@@ -1000,13 +1000,13 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c",
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 22,
|
||||
@@ -1024,11 +1024,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 30
|
||||
},
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1133,7 +1133,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1171,8 +1171,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1203,7 +1203,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1219,7 +1219,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
@@ -1298,12 +1298,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
+15
-15
@@ -995,13 +995,13 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "a76544029263cb74d648e507a64bc91f1222fadeed84f128dc62ed3faafd4b4c",
|
||||
"source_contract_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe",
|
||||
"source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 22,
|
||||
@@ -1019,11 +1019,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 30
|
||||
},
|
||||
"commit": "0979c8753e01e9cbcfbcd9905b160b8f3b6f5372",
|
||||
"commit": "c9a7842991900287ce259e1b624b7f36e3750265",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1128,7 +1128,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 197,
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1166,8 +1166,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921",
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1198,7 +1198,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1214,7 +1214,7 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f",
|
||||
"archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137",
|
||||
"archive_entry_count": 617,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
@@ -1293,12 +1293,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -70,12 +70,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "6721f9af9fc9784b0491f8e73dd92db8a38b3629fc08170ddd3269f2a3c9b48f"
|
||||
"to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "caa815571f7c86fa95447cd7407875eb452cac255c45360db63b017699425921"
|
||||
"to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -16,6 +16,10 @@ REQUIRED_REPORTS = {
|
||||
"interpretation": "reports/skill-interpretation.json",
|
||||
"adoption": "reports/adoption_drift_report.json",
|
||||
"world_class_ledger": "reports/world_class_evidence_ledger.json",
|
||||
"world_class_plan": "reports/world_class_evidence_plan.json",
|
||||
"world_class_intake": "reports/world_class_evidence_intake.json",
|
||||
"world_class_submission_review": "reports/world_class_submission_review.json",
|
||||
"world_class_operator_runbook": "reports/world_class_operator_runbook.json",
|
||||
"skill_os2_coverage": "reports/skill_os2_coverage.json",
|
||||
"review_studio": "reports/review-studio.json",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
@@ -145,6 +149,69 @@ def as_int(value: Any) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]:
|
||||
collection = payload.get(collection_key)
|
||||
if not isinstance(collection, list):
|
||||
return {}
|
||||
keyed: dict[str, dict[str, Any]] = {}
|
||||
for item in collection:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = item.get("key") or item.get("evidence_key")
|
||||
if isinstance(key, str):
|
||||
keyed[key] = item
|
||||
return keyed
|
||||
|
||||
|
||||
def command_key_set(item: dict[str, Any]) -> set[str]:
|
||||
commands = item.get("commands")
|
||||
if isinstance(commands, dict):
|
||||
return {key for key, value in commands.items() if isinstance(key, str) and value}
|
||||
if isinstance(commands, list):
|
||||
keys: set[str] = set()
|
||||
for command in commands:
|
||||
if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"):
|
||||
keys.add(command["key"])
|
||||
return keys
|
||||
return set()
|
||||
|
||||
|
||||
def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
actions = review_studio.get("review_actions")
|
||||
if not isinstance(actions, list):
|
||||
return {}
|
||||
for action in actions:
|
||||
if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence":
|
||||
continue
|
||||
steps = action.get("evidence_steps")
|
||||
if not isinstance(steps, list):
|
||||
return {}
|
||||
keyed: dict[str, dict[str, Any]] = {}
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and isinstance(step.get("key"), str):
|
||||
keyed[step["key"]] = step
|
||||
return keyed
|
||||
return {}
|
||||
|
||||
|
||||
def command_groups_present(command_keys: set[str]) -> dict[str, bool]:
|
||||
return {
|
||||
"prepare_submission": "prepare_submission" in command_keys,
|
||||
"validate_intake": "validate_intake" in command_keys,
|
||||
"submission_review": bool({"submission_review", "review_queue"} & command_keys),
|
||||
"refresh_ledger": "refresh_ledger" in command_keys,
|
||||
"guard_claim": "guard_claim" in command_keys,
|
||||
}
|
||||
|
||||
|
||||
def has_next_action(item: dict[str, Any]) -> bool:
|
||||
for key in ["next_action", "audit_next_action"]:
|
||||
value = item.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def add_check(
|
||||
checks: list[dict[str, Any]],
|
||||
*,
|
||||
@@ -250,6 +317,10 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
interpretation = reports["interpretation"]
|
||||
adoption = reports["adoption"]
|
||||
ledger = reports["world_class_ledger"]
|
||||
world_class_plan = reports["world_class_plan"]
|
||||
world_class_intake = reports["world_class_intake"]
|
||||
world_class_submission_review = reports["world_class_submission_review"]
|
||||
world_class_operator_runbook = reports["world_class_operator_runbook"]
|
||||
coverage = reports["skill_os2_coverage"]
|
||||
review_studio = reports["review_studio"]
|
||||
package_verification = reports["package_verification"]
|
||||
@@ -261,6 +332,10 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
benchmark_summary = nested(benchmark, ["summary"], {})
|
||||
adoption_summary = nested(adoption, ["summary"], {})
|
||||
ledger_summary = nested(ledger, ["summary"], {})
|
||||
plan_summary = nested(world_class_plan, ["summary"], {})
|
||||
intake_summary = nested(world_class_intake, ["summary"], {})
|
||||
submission_review_summary = nested(world_class_submission_review, ["summary"], {})
|
||||
operator_runbook_summary = nested(world_class_operator_runbook, ["summary"], {})
|
||||
coverage_summary = nested(coverage, ["summary"], {})
|
||||
studio_summary = nested(review_studio, ["summary"], {})
|
||||
package_summary = nested(package_verification, ["summary"], {})
|
||||
@@ -510,6 +585,131 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
],
|
||||
detail="The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted.",
|
||||
)
|
||||
ledger_items = keyed_items(ledger, "entries")
|
||||
plan_tasks = keyed_items(world_class_plan, "tasks")
|
||||
intake_checklist = keyed_items(world_class_intake, "operator_checklist")
|
||||
submission_review_items = keyed_items(world_class_submission_review, "items")
|
||||
operator_runbook_items = keyed_items(world_class_operator_runbook, "items")
|
||||
review_action_steps = world_class_review_action_steps(review_studio)
|
||||
pending_world_class_keys = sorted(
|
||||
key for key, item in ledger_items.items() if item.get("status") != "accepted"
|
||||
)
|
||||
expected_command_groups = {
|
||||
key: {
|
||||
"intake": {
|
||||
"prepare_submission": True,
|
||||
"validate_intake": True,
|
||||
"submission_review": True,
|
||||
"refresh_ledger": True,
|
||||
"guard_claim": True,
|
||||
},
|
||||
"operator_runbook": {
|
||||
"prepare_submission": True,
|
||||
"validate_intake": True,
|
||||
"submission_review": True,
|
||||
"refresh_ledger": True,
|
||||
"guard_claim": True,
|
||||
},
|
||||
"review_studio": {
|
||||
"prepare_submission": True,
|
||||
"validate_intake": True,
|
||||
"submission_review": True,
|
||||
"refresh_ledger": True,
|
||||
"guard_claim": True,
|
||||
},
|
||||
}
|
||||
for key in pending_world_class_keys
|
||||
}
|
||||
actual_command_groups = {
|
||||
key: {
|
||||
"intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))),
|
||||
"operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))),
|
||||
"review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))),
|
||||
}
|
||||
for key in pending_world_class_keys
|
||||
}
|
||||
expected_world_class_workflow = {
|
||||
"keys": pending_world_class_keys,
|
||||
"pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None,
|
||||
"human_pending_count": ledger_summary.get("human_pending_count") if isinstance(ledger_summary, dict) else None,
|
||||
"external_pending_count": ledger_summary.get("external_pending_count") if isinstance(ledger_summary, dict) else None,
|
||||
"source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None,
|
||||
"source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None,
|
||||
"source_blocked_count": ledger_summary.get("source_blocked_count") if isinstance(ledger_summary, dict) else None,
|
||||
"plan_keys": pending_world_class_keys,
|
||||
"intake_keys": pending_world_class_keys,
|
||||
"submission_review_keys": pending_world_class_keys,
|
||||
"operator_runbook_keys": pending_world_class_keys,
|
||||
"review_studio_keys": pending_world_class_keys,
|
||||
"intake_ready_to_claim_world_class": False,
|
||||
"submission_review_ready_to_claim_world_class": False,
|
||||
"submission_review_counts_as_completion": False,
|
||||
"operator_runbook_ready_to_claim_world_class": False,
|
||||
"operator_runbook_counts_as_completion": False,
|
||||
"next_actions_present": {key: True for key in pending_world_class_keys},
|
||||
"commands": expected_command_groups,
|
||||
}
|
||||
actual_world_class_workflow = {
|
||||
"keys": pending_world_class_keys,
|
||||
"pending_count": plan_summary.get("task_count") if isinstance(plan_summary, dict) else None,
|
||||
"human_pending_count": plan_summary.get("human_task_count") if isinstance(plan_summary, dict) else None,
|
||||
"external_pending_count": plan_summary.get("external_task_count") if isinstance(plan_summary, dict) else None,
|
||||
"source_check_count": submission_review_summary.get("source_check_count")
|
||||
if isinstance(submission_review_summary, dict)
|
||||
else None,
|
||||
"source_pass_count": submission_review_summary.get("source_pass_count")
|
||||
if isinstance(submission_review_summary, dict)
|
||||
else None,
|
||||
"source_blocked_count": submission_review_summary.get("source_blocked_count")
|
||||
if isinstance(submission_review_summary, dict)
|
||||
else None,
|
||||
"plan_keys": sorted(plan_tasks),
|
||||
"intake_keys": sorted(intake_checklist),
|
||||
"submission_review_keys": sorted(submission_review_items),
|
||||
"operator_runbook_keys": sorted(operator_runbook_items),
|
||||
"review_studio_keys": sorted(review_action_steps),
|
||||
"intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class")
|
||||
if isinstance(intake_summary, dict)
|
||||
else None,
|
||||
"submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class")
|
||||
if isinstance(submission_review_summary, dict)
|
||||
else None,
|
||||
"submission_review_counts_as_completion": submission_review_summary.get(
|
||||
"review_counts_submission_as_completion"
|
||||
)
|
||||
if isinstance(submission_review_summary, dict)
|
||||
else None,
|
||||
"operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class")
|
||||
if isinstance(operator_runbook_summary, dict)
|
||||
else None,
|
||||
"operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion")
|
||||
if isinstance(operator_runbook_summary, dict)
|
||||
else None,
|
||||
"next_actions_present": {
|
||||
key: all(
|
||||
has_next_action(collection.get(key, {}))
|
||||
for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps]
|
||||
)
|
||||
for key in pending_world_class_keys
|
||||
},
|
||||
"commands": actual_command_groups,
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="world-class-evidence-workflow-coverage",
|
||||
label="World-class evidence workflows cover every pending ledger entry",
|
||||
expected=expected_world_class_workflow,
|
||||
actual=actual_world_class_workflow,
|
||||
paths=[
|
||||
REQUIRED_REPORTS["world_class_ledger"],
|
||||
REQUIRED_REPORTS["world_class_plan"],
|
||||
REQUIRED_REPORTS["world_class_intake"],
|
||||
REQUIRED_REPORTS["world_class_submission_review"],
|
||||
REQUIRED_REPORTS["world_class_operator_runbook"],
|
||||
REQUIRED_REPORTS["review_studio"],
|
||||
],
|
||||
detail="Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion.",
|
||||
)
|
||||
skill_os2_review = text_reports.get("skill_os2_review", "")
|
||||
ci_target_count = ci_default_target_count(skill_dir / "scripts" / "ci_test.py")
|
||||
expected_review_snippets = [
|
||||
|
||||
@@ -17,6 +17,10 @@ REPORT_FILES = [
|
||||
"reports/skill-interpretation.html",
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
"reports/world_class_evidence_plan.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_submission_review.json",
|
||||
"reports/world_class_operator_runbook.json",
|
||||
"reports/skill_os2_coverage.json",
|
||||
"reports/review-studio.json",
|
||||
"reports/package_verification.json",
|
||||
@@ -45,6 +49,11 @@ def refresh_embedded_reports() -> None:
|
||||
script_names = [
|
||||
"render_benchmark_reproducibility.py",
|
||||
"render_skill_os2_coverage.py",
|
||||
"render_world_class_evidence_plan.py",
|
||||
"render_world_class_evidence_ledger.py",
|
||||
"render_world_class_evidence_intake.py",
|
||||
"render_world_class_submission_review.py",
|
||||
"render_world_class_operator_runbook.py",
|
||||
"render_world_class_claim_guard.py",
|
||||
"render_skill_overview.py",
|
||||
"render_skill_interpretation.py",
|
||||
@@ -112,7 +121,7 @@ def main() -> None:
|
||||
assert payload["ok"] is True, payload
|
||||
assert payload["summary"]["decision"] == "consistent", payload
|
||||
assert payload["summary"]["fail_count"] == 0, payload
|
||||
assert payload["summary"]["check_count"] >= 28, payload
|
||||
assert payload["summary"]["check_count"] >= 29, payload
|
||||
checks = {item["key"]: item for item in payload["checks"]}
|
||||
assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"]
|
||||
assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"]
|
||||
@@ -121,6 +130,9 @@ def main() -> None:
|
||||
assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[
|
||||
"claim-guard-package-runtime-surface"
|
||||
]
|
||||
assert checks["world-class-evidence-workflow-coverage"]["status"] == "pass", checks[
|
||||
"world-class-evidence-workflow-coverage"
|
||||
]
|
||||
assert checks["skill-os-2-review-current-evidence"]["status"] == "pass", checks[
|
||||
"skill-os-2-review-current-evidence"
|
||||
]
|
||||
@@ -184,6 +196,36 @@ def main() -> None:
|
||||
assert claim_guard_drift_checks["claim-guard-package-runtime-surface"]["status"] == "fail", (
|
||||
claim_guard_drift_checks["claim-guard-package-runtime-surface"]
|
||||
)
|
||||
|
||||
workflow_drift_root = TMP / "workflow-drift-skill"
|
||||
copy_reports(workflow_drift_root)
|
||||
studio_path = workflow_drift_root / "reports" / "review-studio.json"
|
||||
studio = json.loads(studio_path.read_text(encoding="utf-8"))
|
||||
for action in studio["review_actions"]:
|
||||
if action.get("gate_key") == "world-class-evidence":
|
||||
action["evidence_steps"] = [
|
||||
item for item in action["evidence_steps"] if item.get("key") != "native-client-telemetry"
|
||||
]
|
||||
studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
workflow_drift_proc = run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(workflow_drift_root),
|
||||
"--output-json",
|
||||
str(TMP / "workflow_drift.json"),
|
||||
"--output-md",
|
||||
str(TMP / "workflow_drift.md"),
|
||||
"--generated-at",
|
||||
"2026-06-15",
|
||||
]
|
||||
)
|
||||
assert workflow_drift_proc.returncode == 2, workflow_drift_proc.stdout
|
||||
workflow_drift_payload = json.loads(workflow_drift_proc.stdout)
|
||||
workflow_drift_checks = {item["key"]: item for item in workflow_drift_payload["checks"]}
|
||||
assert workflow_drift_checks["world-class-evidence-workflow-coverage"]["status"] == "fail", (
|
||||
workflow_drift_checks["world-class-evidence-workflow-coverage"]
|
||||
)
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user