From 2dc8bad3e02143db969087079317a68a55701747 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sun, 14 Jun 2026 15:07:01 +0800 Subject: [PATCH] chore: refresh evidence runbook artifacts --- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/adoption_drift_report.json | 2 +- reports/architecture_maintainability.json | 28 +- reports/architecture_maintainability.md | 12 +- reports/benchmark_reproducibility.json | 78 +++-- reports/benchmark_reproducibility.md | 45 +-- reports/context_budget.json | 20 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 12 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/package_verification.json | 2 +- reports/package_verification.md | 2 +- reports/python_compatibility.json | 20 +- reports/python_compatibility.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 18 +- reports/review-studio.json | 374 ++++++++++++++++------ reports/review-viewer.json | 65 +++- reports/review_annotations.json | 2 +- reports/runtime_permission_probes.json | 10 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-overview.json | 65 +++- reports/skill_atlas.json | 2 + reports/skill_os2_audit.json | 4 +- reports/skill_os2_audit.md | 4 +- reports/upgrade_check.json | 4 +- reports/world_class_evidence_intake.json | 35 ++ reports/world_class_evidence_intake.md | 39 +++ reports/world_class_evidence_ledger.json | 35 ++ reports/world_class_evidence_ledger.md | 39 +++ reports/world_class_operator_runbook.html | 4 + reports/world_class_operator_runbook.json | 35 ++ reports/world_class_operator_runbook.md | 39 +++ skill_atlas/catalog.json | 2 + 38 files changed, 786 insertions(+), 279 deletions(-) diff --git a/registry/index.json b/registry/index.json index 30903a9..0d9ca92 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 22758f9..f94cdae 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index dccfe43..6c00922 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14T00:00:00Z", + "generated_at": "2026-06-14T07:04:38Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index f8ed61c..8c990e1 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-14", "skill_dir": ".", "summary": { - "python_file_count": 156, - "script_file_count": 97, - "test_file_count": 59, - "internal_module_count": 23, + "python_file_count": 159, + "script_file_count": 99, + "test_file_count": 60, + "internal_module_count": 25, "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, @@ -74,6 +74,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_review_studio.py", + "lines": 695, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_review_viewer.py", "lines": 685, @@ -81,13 +88,6 @@ "severity": "pass", "recommendation": "Split viewer data assembly from HTML section rendering." }, - { - "path": "tests/verify_review_studio.py", - "lines": 680, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/skill_report_layout.py", "lines": 653, @@ -96,9 +96,9 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/render_reference_synthesis.py", - "lines": 644, - "kind": "cli-script", + "path": "scripts/review_studio_gates.py", + "lines": 645, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 968bc6a..cf073bf 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,10 +5,10 @@ Generated at: `2026-06-14` ## Summary - decision: `pass` -- python files: `156` -- scripts: `97` -- tests: `59` -- internal modules: `23` +- python files: `159` +- scripts: `99` +- tests: `60` +- internal modules: `25` - CLI scripts: `76` - Yao CLI command handlers: `34` - largest file lines: `891` @@ -33,10 +33,10 @@ No file-size hotspots found. | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | +| `tests/verify_review_studio.py` | `695` | `test` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | -| `tests/verify_review_studio.py` | `680` | `test` | `pass` | | `scripts/skill_report_layout.py` | `653` | `internal-module` | `pass` | -| `scripts/render_reference_synthesis.py` | `644` | `cli-script` | `pass` | +| `scripts/review_studio_gates.py` | `645` | `internal-module` | `pass` | ## Release Rule diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 994cf2d..537e626 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,23 +3,36 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 38, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83", - "source_contract_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "evidence_bundle_sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5", + "source_contract_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 21, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 38 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "ready": false, + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83" + "sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -138,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7966, - "sha256": "a59b750aebed61481e2fa9a66bd196f1a8c87e5d5857c4e615c85eab156594ad" + "sha256": "648888efd4711dca8867f3362fcc032e0d339bec1ed909f8361ed6d80c7c2d37" }, { "label": "blind_review", @@ -173,28 +187,28 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 100003, - "sha256": "0415c413f6f024e413df0afb4adea17e2c2efc1e3a37d2e8689e6a05241e153e" + "sha256": "7485414e2c21348b67f13848eb5f71879bffb1790c6f210f2504ce776c4ba382" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 20747, - "sha256": "ae16e17266e4e7fed08f4423f8abca7902b0d1cc0839b5692925c62463b72e58" + "bytes": 21114, + "sha256": "293ad17a317463460b81ec176bff3b9a47bb1dde668e26f6393470022d4ca61e" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "a3e00a3b9653da411f20ec24d488067efca021d1270b528f8e2facd1e9ddf743" + "sha256": "d9260f9f8065cf65c75a726e1ec46b27ba541f9841eba6a8cad146b452edde1e" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "3ca24a8ae4b7a3ab68398022754abdec97a5a4b8d451b969d9e7d062244a9617" + "sha256": "6bd47290c4ed45a58ed36f5b9382b0962cab49303269e29bdc4d11b78769615e" }, { "label": "install_simulation", @@ -208,7 +222,7 @@ "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14309, - "sha256": "6bb2dcb0e1e590deacd04c28a86bc74769c82810ecca6a5f660d6615e8db7c18" + "sha256": "e315f88fdec3c5355d53c73372df62df52ed912dd8f920624a62aa66b21168fc" }, { "label": "world_class_evidence_plan", @@ -221,15 +235,15 @@ "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 16763, - "sha256": "7b3d12cd30870cb361b7bb09e43d5ea88fd5bbe35a6d78f91303862a71489ff2" + "bytes": 20006, + "sha256": "9cd09dbeb451d9ec052b77cb41e02389c7d05d50bc1b3f12bd5185ccb0b44220" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 15552, - "sha256": "8f7f9db6f8999ba65078968f8e8383ad4a385000d60bc83ff822a3ca582cc6ce" + "bytes": 18865, + "sha256": "3467248c7c40f71b3f3ced7894f48b72195c325085e8fa384dc04e256d63ee46" }, { "label": "world_class_submission_review", @@ -242,22 +256,22 @@ "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 20738, - "sha256": "443e0335d2703160a41bb6dbd077268f889c0e55e27ca15483d17a3b635213a2" + "bytes": 24021, + "sha256": "5252fe6901ed2c3e2d40b0fe4380f6d981c1193b0f98fdebad24915e63f48eb2" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 12104, - "sha256": "4bd789775307e1b7730be09b0c46d35f678cdee67ef2c7ea29a81c97dd446e24" + "bytes": 15080, + "sha256": "4cdb9cf51b11a751322f7edf54da1f4eee00a4228e41a51875e2284f345e25c3" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 17616, - "sha256": "b94a0e9d92b6609eb6c06e608b52aa17174638bc1422e3b3d306521906bccb01" + "bytes": 21030, + "sha256": "04cc091b113f018072d07a1d71ad142773c415a49e633493689903e7b399b5e6" }, { "label": "world_class_claim_guard", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 35996bb..092ef7a 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,19 +1,19 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `4e9a516485753462a803e618df8e8eb8a22184b0` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83` +Commit: `44fe310092906bdef9f775132818a6a4bd59bcd0` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `24` - missing artifacts: `0` -- source contract sha256: `23d757678bc6` -- archive sha256: `2a3600e27e24` +- source contract sha256: `ad4a46c00c94` +- archive sha256: `d7457b52e379` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `21` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `38` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `24` / `24` -- sha256: `0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83` +- sha256: `a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5` ## Methodology Sections @@ -72,24 +73,24 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `a59b750aebed` | +| output_execution | `reports/output_execution_runs.json` | present | `648888efd471` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `0415c413f6f0` | -| python_compatibility | `reports/python_compatibility.json` | present | `ae16e17266e4` | -| registry_audit | `reports/registry_audit.json` | present | `a3e00a3b9653` | -| package_verification | `reports/package_verification.json` | present | `3ca24a8ae4b7` | +| trust_report | `reports/security_trust_report.json` | present | `7485414e2c21` | +| python_compatibility | `reports/python_compatibility.json` | present | `293ad17a3174` | +| registry_audit | `reports/registry_audit.json` | present | `d9260f9f8065` | +| package_verification | `reports/package_verification.json` | present | `6bd47290c4ed` | | install_simulation | `reports/install_simulation.json` | present | `f0d1820d3da2` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `6bb2dcb0e1e5` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `e315f88fdec3` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `164803bb1cea` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `7b3d12cd3087` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `8f7f9db6f899` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `9cd09dbeb451` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `3467248c7c40` | | world_class_submission_review | `reports/world_class_submission_review.json` | present | `a6d609911ed5` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `443e0335d270` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `4bd789775307` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `b94a0e9d92b6` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `5252fe6901ed` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `4cdb9cf51b11` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `04cc091b113f` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` | ## Reproduction Commands diff --git a/reports/context_budget.json b/reports/context_budget.json index 550d9ac..a01da1d 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1205660, + "other_text_tokens": 1231831, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1206427, - "deferred_resource_tokens": 388370, + "estimated_total_text_tokens": 1232598, + "deferred_resource_tokens": 394915, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97 + "estimated_tokens": 347179, + "file_count": 99 }, { "path": "references", @@ -31,8 +31,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97 + "estimated_tokens": 347179, + "file_count": 99 } ], "deferred_resource_governance": { @@ -54,14 +54,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97, + "estimated_tokens": 347179, + "file_count": 99, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 511, + "relevant_file_count": 518, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 diff --git a/reports/context_budget.md b/reports/context_budget.md index e64f367..c9db4fa 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 388370 | `governed` | scripts:340634 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 394915 | `governed` | scripts:347179 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index bc41f9b..2c0fb7e 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-14", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 388370, + "deferred_resource_tokens": 394915, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97 + "estimated_tokens": 347179, + "file_count": 99 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97, + "estimated_tokens": 347179, + "file_count": 99, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 7cd6f80..d2d2454 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.59, + "duration_ms": 29.29, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.08, + "duration_ms": 28.1, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.46, + "duration_ms": 30.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.55, + "duration_ms": 45.97, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.18, + "duration_ms": 35.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.3, + "duration_ms": 50.03, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.55, + "duration_ms": 71.34, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.32, + "duration_ms": 38.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.75, + "duration_ms": 35.01, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.17, + "duration_ms": 43.11, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index d898b38..b886e9c 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 31.59 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 32.08 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 33.46 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 35.55 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 31.18 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 31.3 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.55 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.32 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.75 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 32.17 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 29.29 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.1 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 30.91 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 45.97 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 35.68 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 50.03 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 71.34 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 38.61 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 35.01 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 43.11 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index e840fbb..3b4dddf 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "archive_entry_count": 579, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index f5baac9..c3dee47 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc` +- Archive SHA256: `d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index f70fb9b..5ebf6bc 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 159, + "file_count": 162, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -130,6 +130,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/import_telemetry_events.py", "ok": true, @@ -580,6 +586,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_source_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/yao.py", "ok": true, @@ -694,6 +706,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_install_simulation.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 8f0e1c6..6fb062b 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-14` - decision: `pass` - target python: `3.11` -- files scanned: `159` +- files scanned: `162` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 4673595..f065158 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 1f13e16..f4b614d 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed` -- Archive SHA256: `2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc` +- Package SHA256: `ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f` +- Archive SHA256: `d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index c2a840e..151c71c 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -543,12 +543,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

99 scripts scanned; secrets found

Py Compat0

159 files scanned for Python 3.11

Arch Debt0

891 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

579 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

99 scripts scanned; secrets found

Py Compat0

162 files scanned for Python 3.11

Arch Debt0

891 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

579 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 388370/120000; top deferred scripts 340634; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 99 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 159 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

156 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 394915/120000; top deferred scripts 347179; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 99 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 162 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

159 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -610,17 +610,17 @@
-

上下文

initial load 960/1000; deferred 388370/120000; top deferred scripts 340634; resource governance governed; quality density 135.4

+

上下文

initial load 960/1000; deferred 394915/120000; top deferred scripts 347179; resource governance governed; quality density 135.4

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
99
网络脚本
3
Help 失败
0
包体哈希
23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed
+

信任报告

Secret
0
脚本数
99
网络脚本
3
Help 失败
0
包体哈希
ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
159
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
162
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线;本地更高版本允许的新语法不能绕过兼容门禁。

@@ -673,8 +673,8 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
+

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
@@ -694,12 +694,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
579
失败数
0
警告数
0
归档哈希
2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
579
失败数
0
警告数
0
归档哈希
d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81
diff --git a/reports/review-studio.json b/reports/review-studio.json index f168c95..a27190d 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -43,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 960/1000; deferred 388370/120000; top deferred scripts 340634; resource governance governed; quality density 135.4", + "detail": "initial load 960/1000; deferred 394915/120000; top deferred scripts 347179; resource governance governed; quality density 135.4", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 159 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 162 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "156 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers", + "detail": "159 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -1332,13 +1332,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83", - "source_contract_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "evidence_bundle_sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5", + "source_contract_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 21, @@ -1356,11 +1356,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 38 }, - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1465,7 +1465,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 183, - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" }, "skill_atlas": { "skill_count": 12, @@ -1503,8 +1503,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -1535,7 +1535,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1551,7 +1551,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "archive_entry_count": 579, "failure_count": 0, "warning_count": 0 @@ -1630,12 +1630,12 @@ { "field": "archive_sha256", "from": "", - "to": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "to": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "to": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, @@ -1961,6 +1961,12 @@ "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "provider-backed model run", "observed timing", @@ -2052,6 +2058,16 @@ "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real reviewer identity", "blind A/B decisions", @@ -2157,6 +2173,16 @@ "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", @@ -2255,6 +2281,15 @@ "source_accepted": false, "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real external client source", "metadata-only event", @@ -3967,7 +4002,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.59, + "duration_ms": 29.29, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3995,7 +4030,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.08, + "duration_ms": 28.1, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4018,7 +4053,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.46, + "duration_ms": 30.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4046,7 +4081,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.55, + "duration_ms": 45.97, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4069,7 +4104,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.18, + "duration_ms": 35.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4097,7 +4132,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.3, + "duration_ms": 50.03, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4120,7 +4155,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.55, + "duration_ms": 71.34, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4147,7 +4182,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.32, + "duration_ms": 38.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4170,7 +4205,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.75, + "duration_ms": 35.01, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4199,7 +4234,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.17, + "duration_ms": 43.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4920,23 +4955,36 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 38, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83", - "source_contract_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "evidence_bundle_sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5", + "source_contract_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 21, @@ -4954,14 +5002,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 38 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -4970,10 +5019,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "ready": false, + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -4981,7 +5030,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83" + "sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5055,7 +5104,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7966, - "sha256": "a59b750aebed61481e2fa9a66bd196f1a8c87e5d5857c4e615c85eab156594ad" + "sha256": "648888efd4711dca8867f3362fcc032e0d339bec1ed909f8361ed6d80c7c2d37" }, { "label": "blind_review", @@ -5090,28 +5139,28 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 100003, - "sha256": "0415c413f6f024e413df0afb4adea17e2c2efc1e3a37d2e8689e6a05241e153e" + "sha256": "7485414e2c21348b67f13848eb5f71879bffb1790c6f210f2504ce776c4ba382" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 20747, - "sha256": "ae16e17266e4e7fed08f4423f8abca7902b0d1cc0839b5692925c62463b72e58" + "bytes": 21114, + "sha256": "293ad17a317463460b81ec176bff3b9a47bb1dde668e26f6393470022d4ca61e" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "a3e00a3b9653da411f20ec24d488067efca021d1270b528f8e2facd1e9ddf743" + "sha256": "d9260f9f8065cf65c75a726e1ec46b27ba541f9841eba6a8cad146b452edde1e" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "3ca24a8ae4b7a3ab68398022754abdec97a5a4b8d451b969d9e7d062244a9617" + "sha256": "6bd47290c4ed45a58ed36f5b9382b0962cab49303269e29bdc4d11b78769615e" }, { "label": "install_simulation", @@ -5125,7 +5174,7 @@ "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14309, - "sha256": "6bb2dcb0e1e590deacd04c28a86bc74769c82810ecca6a5f660d6615e8db7c18" + "sha256": "e315f88fdec3c5355d53c73372df62df52ed912dd8f920624a62aa66b21168fc" }, { "label": "world_class_evidence_plan", @@ -5138,15 +5187,15 @@ "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 16763, - "sha256": "7b3d12cd30870cb361b7bb09e43d5ea88fd5bbe35a6d78f91303862a71489ff2" + "bytes": 20006, + "sha256": "9cd09dbeb451d9ec052b77cb41e02389c7d05d50bc1b3f12bd5185ccb0b44220" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 15552, - "sha256": "8f7f9db6f8999ba65078968f8e8383ad4a385000d60bc83ff822a3ca582cc6ce" + "bytes": 18865, + "sha256": "3467248c7c40f71b3f3ced7894f48b72195c325085e8fa384dc04e256d63ee46" }, { "label": "world_class_submission_review", @@ -5159,22 +5208,22 @@ "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 20738, - "sha256": "443e0335d2703160a41bb6dbd077268f889c0e55e27ca15483d17a3b635213a2" + "bytes": 24021, + "sha256": "5252fe6901ed2c3e2d40b0fe4380f6d981c1193b0f98fdebad24915e63f48eb2" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 12104, - "sha256": "4bd789775307e1b7730be09b0c46d35f678cdee67ef2c7ea29a81c97dd446e24" + "bytes": 15080, + "sha256": "4cdb9cf51b11a751322f7edf54da1f4eee00a4228e41a51875e2284f345e25c3" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 17616, - "sha256": "b94a0e9d92b6609eb6c06e608b52aa17174638bc1422e3b3d306521906bccb01" + "bytes": 21030, + "sha256": "04cc091b113f018072d07a1d71ad142773c415a49e633493689903e7b399b5e6" }, { "label": "world_class_claim_guard", @@ -10401,7 +10450,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "tests/tmp_review_studio/dist", + "package_dir": "dist", "expected_capabilities": [ "file_write", "network", @@ -10441,7 +10490,7 @@ { "target": "openai", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", + "adapter": "dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10538,7 +10587,7 @@ { "target": "claude", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", + "adapter": "dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10620,7 +10669,7 @@ { "target": "generic", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", + "adapter": "dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10702,7 +10751,7 @@ { "target": "vscode", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", + "adapter": "dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10813,7 +10862,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 183, - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" }, "failures": [], "warnings": [], @@ -13305,7 +13354,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 159, + "file_count": 162, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -13430,6 +13479,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/import_telemetry_events.py", "ok": true, @@ -13880,6 +13935,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_source_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/yao.py", "ok": true, @@ -13994,6 +14055,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_install_simulation.py", "ok": true, @@ -14289,10 +14356,10 @@ "generated_at": "2026-06-14", "skill_dir": ".", "summary": { - "python_file_count": 156, - "script_file_count": 97, - "test_file_count": 59, - "internal_module_count": 23, + "python_file_count": 159, + "script_file_count": 99, + "test_file_count": 60, + "internal_module_count": 25, "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, @@ -14359,6 +14426,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_review_studio.py", + "lines": 695, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/render_review_viewer.py", "lines": 685, @@ -14366,13 +14440,6 @@ "severity": "pass", "recommendation": "Split viewer data assembly from HTML section rendering." }, - { - "path": "tests/verify_review_studio.py", - "lines": 680, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/skill_report_layout.py", "lines": 653, @@ -14381,9 +14448,9 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/render_reference_synthesis.py", - "lines": 644, - "kind": "cli-script", + "path": "scripts/review_studio_gates.py", + "lines": 645, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } @@ -14403,16 +14470,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1205660, + "other_text_tokens": 1231831, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1206427, - "deferred_resource_tokens": 388370, + "estimated_total_text_tokens": 1232598, + "deferred_resource_tokens": 394915, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97 + "estimated_tokens": 347179, + "file_count": 99 }, { "path": "references", @@ -14428,8 +14495,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97 + "estimated_tokens": 347179, + "file_count": 99 } ], "deferred_resource_governance": { @@ -14451,14 +14518,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 340634, - "file_count": 97, + "estimated_tokens": 347179, + "file_count": 99, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 511, + "relevant_file_count": 518, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 @@ -15337,6 +15404,7 @@ "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", @@ -15412,6 +15480,7 @@ "scripts/validate_skill.py", "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", + "scripts/world_class_source_checks.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", @@ -16314,7 +16383,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14T00:00:00Z", + "generated_at": "2026-06-14T07:04:38Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -16479,7 +16548,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -16534,6 +16603,12 @@ "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "provider-backed model run", "observed timing", @@ -16625,6 +16700,16 @@ "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real reviewer identity", "blind A/B decisions", @@ -16730,6 +16815,16 @@ "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", @@ -16828,6 +16923,15 @@ "source_accepted": false, "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real external client source", "metadata-only event", @@ -17042,6 +17146,12 @@ "observed timing", "observed token metadata" ], + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/output_execution_runs.json summary.model_executed_count > 0", "reports/output_execution_runs.json summary.timing_observed_count > 0", @@ -17103,6 +17213,16 @@ "blind A/B decisions", "answer key unopened until decisions exist" ], + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", @@ -17168,6 +17288,16 @@ "native enforcement flag or externally accepted guard proof", "residual risk retained for fallback targets" ], + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", "reports/runtime_permission_probes.json summary.failure_count == 0", @@ -17231,6 +17361,15 @@ "metadata-only event", "local-first import path" ], + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -17635,6 +17774,12 @@ "source_accepted": false, "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 0; token-observed 0", + "execution_runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", @@ -17740,6 +17885,16 @@ "source_accepted": false, "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "current": "0/5 decisions; pending 5", + "execution_runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/human-adjudication.json", "template_path": "evidence/world_class/templates/human-adjudication.intake.json", @@ -17859,6 +18014,16 @@ "source_accepted": false, "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "current": "native-enforced targets 0; installer-enforced targets 4", + "execution_runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", @@ -17970,6 +18135,15 @@ "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", "current": "external source events 0; adoption samples 0", + "execution_runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", @@ -18445,8 +18619,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -18477,7 +18651,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -18502,7 +18676,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, @@ -18525,7 +18699,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "archive_entry_count": 579, "failure_count": 0, "warning_count": 0 @@ -19533,12 +19707,12 @@ { "field": "archive_sha256", "from": "", - "to": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "to": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "to": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 5154573..55d260f 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -919,13 +919,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83", - "source_contract_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "evidence_bundle_sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5", + "source_contract_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 21, @@ -943,11 +943,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 38 }, - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1052,7 +1052,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 183, - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" }, "skill_atlas": { "skill_count": 12, @@ -1090,8 +1090,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -1122,7 +1122,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1138,7 +1138,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "archive_entry_count": 579, "failure_count": 0, "warning_count": 0 @@ -1217,12 +1217,12 @@ { "field": "archive_sha256", "from": "", - "to": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "to": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "to": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, @@ -1548,6 +1548,12 @@ "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "provider-backed model run", "observed timing", @@ -1639,6 +1645,16 @@ "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real reviewer identity", "blind A/B decisions", @@ -1744,6 +1760,16 @@ "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", @@ -1842,6 +1868,15 @@ "source_accepted": false, "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real external client source", "metadata-only event", diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 66f2b1d..76fadc1 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json index 59c9a1a..9c2ffd6 100644 --- a/reports/runtime_permission_probes.json +++ b/reports/runtime_permission_probes.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "tests/tmp_review_studio/dist", + "package_dir": "dist", "expected_capabilities": [ "file_write", "network", @@ -42,7 +42,7 @@ { "target": "openai", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", + "adapter": "dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -139,7 +139,7 @@ { "target": "claude", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", + "adapter": "dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -221,7 +221,7 @@ { "target": "generic", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", + "adapter": "dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -303,7 +303,7 @@ { "target": "vscode", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", + "adapter": "dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 80addf0..907d889 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 183, - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index dfd8a6e..564b7c9 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `183` -- Package SHA256: `23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed` +- Package SHA256: `ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f` ## Failures diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 45cfa6b..f42dd5e 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -918,13 +918,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0b7c79cf4a585bdbd7acd1b328fb6dbceee35ef27999e609b0cf0a2e3c2ece83", - "source_contract_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "evidence_bundle_sha256": "a04ceeb7551b598338dc4c9306a64e770d0afa476d09bae2176c411b88336fd5", + "source_contract_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 21, @@ -942,11 +942,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 38 }, - "commit": "4e9a516485753462a803e618df8e8eb8a22184b0", + "commit": "44fe310092906bdef9f775132818a6a4bd59bcd0", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1051,7 +1051,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 183, - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" }, "skill_atlas": { "skill_count": 12, @@ -1089,8 +1089,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed", - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "package_sha256": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, "compatibility": { "openai": "pass", @@ -1121,7 +1121,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1137,7 +1137,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc", + "archive_sha256": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81", "archive_entry_count": 579, "failure_count": 0, "warning_count": 0 @@ -1216,12 +1216,12 @@ { "field": "archive_sha256", "from": "", - "to": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "to": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "to": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, @@ -1547,6 +1547,12 @@ "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "provider-backed model run", "observed timing", @@ -1638,6 +1644,16 @@ "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real reviewer identity", "blind A/B decisions", @@ -1743,6 +1759,16 @@ "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", @@ -1841,6 +1867,15 @@ "source_accepted": false, "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real external client source", "metadata-only event", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index a9d88ae..a96af46 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -79,6 +79,7 @@ "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", @@ -154,6 +155,7 @@ "scripts/validate_skill.py", "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", + "scripts/world_class_source_checks.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index f5db200..f362d45 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 97; help failures 0", + "current": "secrets 0; scripts 99; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 576; install failures 0; permission failures 0", + "current": "zip entries 579; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index a48e7a7..05f9a8e 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-14` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 97; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 99; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 576; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 579; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index afb7bdc..60377bb 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "2a3600e27e24fc5799cce599f82763f78bbc6a682a96f08ecd9034f4510344bc" + "to": "d7457b52e37910862b7519d510fb195fe31da1a4d5f25396237b6be18c323d81" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "23d757678bc61e183af53ac0724e02f44128857f51481984e6b6ffe6575fdfed" + "to": "ad4a46c00c94b3c7cb75b8b6e65623b14571bb117000bf69b82be1b8cf330d6f" } ] }, diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 4f9550e..ac129f4 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -112,6 +112,12 @@ "observed timing", "observed token metadata" ], + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/output_execution_runs.json summary.model_executed_count > 0", "reports/output_execution_runs.json summary.timing_observed_count > 0", @@ -173,6 +179,16 @@ "blind A/B decisions", "answer key unopened until decisions exist" ], + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", @@ -238,6 +254,16 @@ "native enforcement flag or externally accepted guard proof", "residual risk retained for fallback targets" ], + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", "reports/runtime_permission_probes.json summary.failure_count == 0", @@ -301,6 +327,15 @@ "metadata-only event", "local-first import path" ], + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index b73561c..a77f0ec 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -81,6 +81,13 @@ This report validates the intake contract for human and external evidence. A val - Do not commit provider credentials or environment dumps. - The output execution report records output hashes and aggregate run metadata, not raw provider prompts. +#### Source Runbook + +- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Human Adjudication - readiness: `awaiting-submission` @@ -122,6 +129,17 @@ This report validates the intake contract for human and external evidence. A val - Reviewer decisions should not include raw user data or private customer detail. - Keep the answer key separate until after decisions are recorded. +#### Source Runbook + +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- `python3 scripts/adjudicate_output_review.py --write-template` +- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- `python3 scripts/yao.py output-review` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Native Permission Enforcement - readiness: `awaiting-submission` @@ -164,6 +182,17 @@ This report validates the intake contract for human and external evidence. A val - Do not mark native_enforcement true for metadata-only fallbacks. - Keep residual risks visible for targets that still rely on operator enforcement. +#### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip` +- `python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation` +- `python3 scripts/yao.py runtime-permissions . --package-dir dist` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Native Client Telemetry - readiness: `awaiting-submission` @@ -203,6 +232,16 @@ This report validates the intake contract for human and external evidence. A val - Telemetry must remain metadata-only and local-first. - Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. +#### Source Runbook + +- `python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///` +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl` +- `python3 scripts/yao.py skill-atlas --workspace-root .` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ## Boundary - Templates and planned work do not count as accepted evidence. diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index 9337b8d..6d4d724 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -33,6 +33,12 @@ "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "provider-backed model run", "observed timing", @@ -124,6 +130,16 @@ "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real reviewer identity", "blind A/B decisions", @@ -229,6 +245,16 @@ "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", @@ -327,6 +353,15 @@ "source_accepted": false, "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "provenance_requirements": [ "real external client source", "metadata-only event", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index c478e83..6a46f8d 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -45,6 +45,13 @@ This ledger records the current evidence state. It requires both passing source - observed timing - observed token metadata +### Source Runbook + +- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Source Evidence Checks | Check | Current | Expected | Status | @@ -79,6 +86,17 @@ This ledger records the current evidence state. It requires both passing source - blind A/B decisions - answer key unopened until decisions exist +### Source Runbook + +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- `python3 scripts/adjudicate_output_review.py --write-template` +- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- `python3 scripts/yao.py output-review` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Source Evidence Checks | Check | Current | Expected | Status | @@ -114,6 +132,17 @@ This ledger records the current evidence state. It requires both passing source - native enforcement flag or externally accepted guard proof - residual risk retained for fallback targets +### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip` +- `python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation` +- `python3 scripts/yao.py runtime-permissions . --package-dir dist` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Source Evidence Checks | Check | Current | Expected | Status | @@ -148,6 +177,16 @@ This ledger records the current evidence state. It requires both passing source - metadata-only event - local-first import path +### Source Runbook + +- `python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///` +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl` +- `python3 scripts/yao.py skill-atlas --workspace-root .` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + ### Source Evidence Checks | Check | Current | Expected | Status | diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 1b5695e..be9f267 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -69,6 +69,7 @@
Blocked
2
Submission
evidence/world_class/submissions/provider-holdout.json
+

Source Runbook

  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • provider-backed model run
  • observed timing
  • observed token metadata
@@ -86,6 +87,7 @@
Blocked
2
Submission
evidence/world_class/submissions/human-adjudication.json
+

Source Runbook

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
@@ -103,6 +105,7 @@
Blocked
1
Submission
evidence/world_class/submissions/native-permission-enforcement.json
+

Source Runbook

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
@@ -120,6 +123,7 @@
Blocked
2
Submission
evidence/world_class/submissions/native-client-telemetry.json
+

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • real external client source
  • metadata-only event
  • local-first import path
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index 3f5d816..2251e15 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -34,6 +34,12 @@ "source_accepted": false, "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 0; token-observed 0", + "execution_runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", @@ -139,6 +145,16 @@ "source_accepted": false, "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "current": "0/5 decisions; pending 5", + "execution_runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/human-adjudication.json", "template_path": "evidence/world_class/templates/human-adjudication.intake.json", @@ -258,6 +274,16 @@ "source_accepted": false, "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "current": "native-enforced targets 0; installer-enforced targets 4", + "execution_runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", @@ -369,6 +395,15 @@ "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", "current": "external source events 0; adoption samples 0", + "execution_runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index c087c16..5852016 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -38,6 +38,13 @@ This runbook coordinates evidence collection only. It does not accept submission - submission: `evidence/world_class/submissions/provider-holdout.json` - template: `evidence/world_class/templates/provider-holdout.intake.json` +### Source Runbook + +- YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + ### Commands - prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` @@ -95,6 +102,17 @@ This runbook coordinates evidence collection only. It does not accept submission - submission: `evidence/world_class/submissions/human-adjudication.json` - template: `evidence/world_class/templates/human-adjudication.intake.json` +### Source Runbook + +- python3 scripts/yao.py output-review-kit --write-template +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- python3 scripts/adjudicate_output_review.py --write-template +- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- python3 scripts/yao.py output-review +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + ### Commands - prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` @@ -155,6 +173,17 @@ This runbook coordinates evidence collection only. It does not accept submission - submission: `evidence/world_class/submissions/native-permission-enforcement.json` - template: `evidence/world_class/templates/native-permission-enforcement.intake.json` +### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip +- python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation +- python3 scripts/yao.py runtime-permissions . --package-dir dist +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + ### Commands - prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` @@ -214,6 +243,16 @@ This runbook coordinates evidence collection only. It does not accept submission - submission: `evidence/world_class/submissions/native-client-telemetry.json` - template: `evidence/world_class/templates/native-client-telemetry.intake.json` +### Source Runbook + +- python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:/// +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl +- python3 scripts/yao.py skill-atlas --workspace-root . +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + ### Commands - prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index b07477c..95f4ae1 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -35,6 +35,7 @@ "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", @@ -110,6 +111,7 @@ "scripts/validate_skill.py", "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", + "scripts/world_class_source_checks.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py",