diff --git a/evidence/world_class/templates/provider-holdout.intake.json b/evidence/world_class/templates/provider-holdout.intake.json index f8faf6e3..fffc33c5 100644 --- a/evidence/world_class/templates/provider-holdout.intake.json +++ b/evidence/world_class/templates/provider-holdout.intake.json @@ -18,7 +18,10 @@ "provenance": { "provider": "openai", "model": "gpt-4.1-mini", - "run_command": "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "run_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "credential_env": "OPENAI_API_KEY", + "model_env": "YAO_OUTPUT_EVAL_MODEL", + "model_default": "gpt-4.1-mini", "credential_material_committed": false }, "privacy": { diff --git a/registry/index.json b/registry/index.json index 821559be..deb93c8c 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,6 +1,6 @@ { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 7c7784df..091a666d 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,7 +16,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -53,5 +53,5 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" } diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 71fe0fda..14f57abc 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-16T00:06:24Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -25,14 +25,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -41,7 +41,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -53,31 +53,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 0, - "accepted": 0, + "adoption_events": 1, + "accepted": 1, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 0 + "adoption_rate": 100.0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "review_event", + "event": "skill_activation", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "manual", - "outcome": "reviewed", + "activation_type": "explicit", + "outcome": "accepted", "failure_type": "none", - "timestamp": "2026-06-13T12:00:00Z" + "timestamp": "2026-06-13T10:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "reports/telemetry_events.jsonl", + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md index 91b31983..206372c9 100644 --- a/reports/adoption_drift_report.md +++ b/reports/adoption_drift_report.md @@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Summary - Events: `1` -- Adoption samples: `0` -- Activation events: `0` -- Adoption rate: `0` +- Adoption samples: `1` +- Activation events: `1` +- Adoption rate: `100.0` - Missed trigger signals: `0` - Bad output signals: `0` - Script error signals: `0` @@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, | Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 | +| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 | ## Next Iteration Candidates @@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Recent Metadata Events -- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none` +- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none` diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index a11d367f..aa6895cc 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "python_file_count": 191, @@ -29,6 +29,13 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 874, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -36,16 +43,9 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 867, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", - "lines": 834, + "lines": 836, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -115,6 +115,13 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 874, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -122,16 +129,9 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 867, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", - "lines": 834, + "lines": 836, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 577048d0..3d9b3475 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,6 +1,6 @@ # Architecture Maintainability -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary @@ -30,9 +30,9 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | -| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `tests/verify_review_studio.py` | `834` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_review_studio.py` | `836` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/render_evidence_consistency.py` | `808` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -45,9 +45,9 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | `pass` | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | `pass` | -| `tests/verify_review_studio.py` | `834` | `test` | `pass` | +| `tests/verify_review_studio.py` | `836` | `test` | `pass` | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | `pass` | | `scripts/render_evidence_consistency.py` | `808` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 6ebcec65..332952f1 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,22 +3,35 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 64, + "sample": [ + " M evidence/world_class/templates/provider-holdout.intake.json", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -34,29 +47,30 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (6/13 pass, 7 blocked)" + "world-class source checks are not all accepted (7/13 pass, 6 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "ready": false, + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1" + "sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -138,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "ab66dfd1a1f2fca481752aa164271a8c4f419cab9653f45dd63d1c73011713bb" + "sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d" }, { "label": "blind_review", @@ -173,21 +187,21 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 118486, - "sha256": "ca53fbf5fa66f574d34603b160c3c3a264b015ba2722deb930b5accb94b52af8" + "sha256": "390ddd84dcfb27f7d1c6a3e6216c526ff1ec1ca86c11ea22dfdcdc0403e71cda" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 25199, - "sha256": "18c8a142c8108071bc453367c1fd3589b3368c098d5fc27f4b1d11c19f0bfb92" + "sha256": "dbd04851a3a6a14273a2e62823ee6d1d181d2e91b88c8c34d2a1b5ca284bc35a" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "7f980caf507c7d094401a18194eeeb7480de3a924326b6d0c97e07b33e3fb64f" + "sha256": "7762378b6fe7a3228b24e6a69499efb8e7ea3c394de1c9714c4d886fc38857ff" }, { "label": "package_verification", @@ -200,78 +214,78 @@ "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "aa45be12d2a94664dce95884668839d58808eaad70de90cfb1e96939e9e3a3c0" + "bytes": 8758, + "sha256": "ba0b8b088b64b1b60715075da4502048c50b2b00d922c6cc7bd3456dca148b34" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "a4dfc4e3af1167e3d6ced9444c41bc6cd2133bb67942fedba94c046814bf7866" + "sha256": "6187a8850680edd24e84161003d3d06379f5f9665ad1ed6e509b661283c62a44" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 20554, - "sha256": "38e95161404d2dc9a1c7bf3c1ee213a051ced5444deb01b0a01e557a4321021d" + "bytes": 20828, + "sha256": "af3c3f101ad992820bf36ddb14b02e7404ef16411567c483f86eed530ca45f62" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 21587, - "sha256": "d08019f31a7811ee747392f24a11902f173201054894c0338d80b45543c88d18" + "bytes": 21721, + "sha256": "6c135aced3443b88cc45009957386b566be80fd5c7f261361163dada861dab43" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 19178, - "sha256": "4a9119aba62b7cbc9f6b434c33f58a73d2fd623e1f4ed67c3ca994f23025a317" + "bytes": 19319, + "sha256": "69afd0ff3d49ba9e1ea57248e41cfe62a0abfdf06fd0f997ed3008a5656c2566" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 41071, - "sha256": "37fd4b91f92ceda3a5873431d5629b33c7909c10fbdc465eaad546e45562328f" + "bytes": 41202, + "sha256": "97924dd4be88bedd734bb0530765ce5ea6e10a92d2a17807c14b2aae5b66f960" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 13658, - "sha256": "6aaf868374ba134afbb4d4bf9c0339604ab41b7439379241359b83c279e9207f" + "bytes": 13655, + "sha256": "17def7ba6982d834e13ce3896dd9093019eec3ee8d2fa5b283ef8d6b17b19be0" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 24330, - "sha256": "40d1796964450e55bbaea98e46b0aa3433693e0b532c89b77ce5279c020570ff" + "bytes": 24403, + "sha256": "f2e53f857320f4c98faa8375e6b4758a8b6b291497578ec9adb14baeb6ee169c" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 15360, - "sha256": "ebfa3a4b528044bdf40f8931a246b54eb7cf37afe55bc02cad95f6e04ea4efa0" + "bytes": 15424, + "sha256": "337fb764b98ad2707f5a0d7bba88b9c5401256d7be00f8f3dc478e89e3bcb6c2" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 21284, - "sha256": "eb6c4506b0f25f01eef077c7c65bfe3f3230d3991b0db3befa8798768254373f" + "bytes": 21348, + "sha256": "ca72c65b437b2211d08b9d463c9a33a7993b4e388bb75f62757ead4dfcf81a49" }, { "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, "bytes": 18406, - "sha256": "704aba252d55e9b9f546392ce7f51b91f3b74d3384501d7f80a74e14c5b50acd" + "sha256": "25206d72ff919a2a4ce398b80a602028ab5f6a43648b5d3b690b3fdd02f49280" } ], "missing_artifacts": [], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 2681b29f..22700a8b 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,18 +1,18 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `4b2767900bbbbb962d2b11a92610951c3a0a0f0e` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1` +Commit: `54851019f53f3ee3f508dedfc0daaf192d6bc6ad` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `db36e8f9d7d6` +- source contract sha256: `a5f307ba8e89` - archive sha256: `996e36863203` - output cases: `5` - disclosed failure cases: `3` @@ -20,10 +20,10 @@ Evidence bundle SHA256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- world-class source checks: `6` pass / `13` total; `7` blocked +- world-class source checks: `7` pass / `13` total; `6` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `64` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,22 +35,23 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | -| world-class source checks are not all accepted (6/13 pass, 7 blocked) | +| world-class source checks are not all accepted (7/13 pass, 6 blocked) | ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1` +- sha256: `95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f` ## Methodology Sections @@ -72,26 +73,26 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `ab66dfd1a1f2` | +| output_execution | `reports/output_execution_runs.json` | present | `072e7ff6ef9c` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `ca53fbf5fa66` | -| python_compatibility | `reports/python_compatibility.json` | present | `18c8a142c810` | -| registry_audit | `reports/registry_audit.json` | present | `7f980caf507c` | +| trust_report | `reports/security_trust_report.json` | present | `390ddd84dcfb` | +| python_compatibility | `reports/python_compatibility.json` | present | `dbd04851a3a6` | +| registry_audit | `reports/registry_audit.json` | present | `7762378b6fe7` | | package_verification | `reports/package_verification.json` | present | `bbe73152879c` | -| install_simulation | `reports/install_simulation.json` | present | `aa45be12d2a9` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4dfc4e3af11` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `38e95161404d` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `d08019f31a78` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `4a9119aba62b` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `37fd4b91f92c` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `6aaf868374ba` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `40d179696445` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `ebfa3a4b5280` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `eb6c4506b0f2` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `704aba252d55` | +| install_simulation | `reports/install_simulation.json` | present | `ba0b8b088b64` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `6187a8850680` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `af3c3f101ad9` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `6c135aced344` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `69afd0ff3d49` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `97924dd4be88` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `17def7ba6982` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `f2e53f857320` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `337fb764b98a` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `ca72c65b437b` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `25206d72ff91` | ## Reproduction Commands diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index eeb3e744..78d28a9b 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, diff --git a/reports/context_budget.json b/reports/context_budget.json index ff3cb719..f68f25cb 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1045126, + "other_text_tokens": 1046015, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1045923, - "deferred_resource_tokens": 484750, + "estimated_total_text_tokens": 1046812, + "deferred_resource_tokens": 484786, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index 62ca623e..5817f898 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 484750 | `governed` | scripts:424772 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 484786 | `governed` | scripts:424808 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 358900ea..e4dd2c98 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 484750, + "deferred_resource_tokens": 484786, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 1f554e24..acbf3621 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -207,8 +207,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -248,8 +248,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", - "actual": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "expected": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", + "actual": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -261,30 +261,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -298,13 +298,13 @@ "status": "pass", "expected": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, + "adoption_rate": 100.0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "source_types": { "manual": 1 @@ -312,13 +312,13 @@ }, "actual": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, + "adoption_rate": 100.0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "source_types": { "manual": 1 @@ -341,8 +341,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -353,8 +353,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -374,7 +374,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 13, - "source_pass_count": 6 + "source_pass_count": 7 }, "actual": { "ready": false, @@ -382,7 +382,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 13, - "source_pass_count": 6 + "source_pass_count": 7 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -394,8 +394,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", - "actual": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "expected": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", + "actual": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -407,30 +407,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -444,13 +444,13 @@ "status": "pass", "expected": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, + "adoption_rate": 100.0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "source_types": { "manual": 1 @@ -458,13 +458,13 @@ }, "actual": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, + "adoption_rate": 100.0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "source_types": { "manual": 1 @@ -487,8 +487,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -499,8 +499,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -520,7 +520,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 13, - "source_pass_count": 6 + "source_pass_count": 7 }, "actual": { "ready": false, @@ -528,7 +528,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 13, - "source_pass_count": 6 + "source_pass_count": 7 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -1327,7 +1327,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1386,8 +1386,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -1401,7 +1400,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1460,8 +1459,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -1811,15 +1809,15 @@ "expected": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false }, "actual": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false }, "paths": [ @@ -1835,8 +1833,8 @@ "expected": { "pending_count": 4, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -1844,8 +1842,8 @@ "actual": { "pending_count": 4, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -2099,8 +2097,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "plan_keys": [ "human-adjudication", "native-client-telemetry", @@ -2248,8 +2246,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "plan_keys": [ "human-adjudication", "native-client-telemetry", diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 77949e2e..35db4587 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -1,11 +1,11 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, "archive_entry_count": 647, @@ -26,7 +26,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -272,9 +272,9 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } } diff --git a/reports/install_simulation.md b/reports/install_simulation.md index 6750e007..b0d16dae 100644 --- a/reports/install_simulation.md +++ b/reports/install_simulation.md @@ -1,7 +1,7 @@ # Install Simulation - OK: `True` -- Package directory: `dist` +- Package directory: `tests/tmp_review_studio/dist` - Archive extracted: `True` - Entrypoint loaded: `True` - Manifest loaded: `True` @@ -16,7 +16,7 @@ | Check | Status | Detail | | --- | --- | --- | -| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | +| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | | `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 59ca0b81..ec5cf0ca 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.96, + "duration_ms": 46.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.37, + "duration_ms": 45.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.25, + "duration_ms": 48.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.14, + "duration_ms": 43.77, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.63, + "duration_ms": 43.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.11, + "duration_ms": 40.08, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.98, + "duration_ms": 46.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.13, + "duration_ms": 43.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.48, + "duration_ms": 40.54, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.28, + "duration_ms": 40.25, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 96f53c6f..69660ccc 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 35.96 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 34.37 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 35.25 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 35.14 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 35.63 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 34.11 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 29.98 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 30.13 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 30.48 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 30.28 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 46.38 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 45.36 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 48.62 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.77 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 43.21 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 40.08 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 46.12 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 43.65 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 40.54 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 40.25 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index d7673b36..cc2eb6eb 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index c7b3e7ff..fa4cccc5 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,6 +1,6 @@ # Python Compatibility -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 4bb4130b..e3a1df6e 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,7 +21,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -58,11 +58,11 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 8bed3682..3f15feaa 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,7 +6,7 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336` +- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f` - Archive SHA256: `996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81` - Install simulated: `True` diff --git a/reports/review-studio.html b/reports/review-studio.html index 18201830..dca66210 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -740,22 +740,22 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

122 scripts scanned; secrets found

Py Compat0

194 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

647 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

122 scripts scanned; secrets found

Py Compat0

194 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

647 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据

阻断事项

无。

-

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true
+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 1

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
1 blocked / 2 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -807,12 +807,12 @@
-

上下文

initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
122
网络脚本
3
Help 失败
0
包体哈希
db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336
+

信任报告

Secret
0
脚本数
122
网络脚本
3
Help 失败
0
包体哈希
a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -837,14 +837,14 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

-

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

+

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
-

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
世界级
Writes Source Files
Auto Patch Enabled
-

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
Writes Source Files
Auto Patch Enabled
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
世界级
Writes Source Files
Auto Patch Enabled
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
Writes Source Files
Auto Patch Enabled
@@ -856,19 +856,19 @@

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

-
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
+
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步;计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

提交清单

每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界;只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后,才进入 ledger review。

-
awaiting-submission · external

Provider Holdout

No real evidence submission has been provided yet.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices in the decision template, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
awaiting-submission · external

Provider Holdout

No real evidence submission has been provided yet.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices in the decision template, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -877,12 +877,12 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
+

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (7/13 pass, 6 blocked)
-

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

+

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

证据台账

Ledger Entry Count
4
Accepted Count
0
待审
4
Human Pending Count
1
External Pending Count
3
Overclaim Guard Active
Ready To Claim World Class
diff --git a/reports/review-studio.json b/reports/review-studio.json index f18470ed..fc8aa114 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -43,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -107,7 +107,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md" }, @@ -123,7 +123,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" }, @@ -166,7 +166,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" } @@ -494,9 +494,9 @@ } ], "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", - "python3 scripts/yao.py skill-os2-audit . --generated-at ", - "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields." + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60" ], "provenance_requirements": [ "provider-backed model run", @@ -771,12 +771,12 @@ "category": "external", "status": "pending", "readiness": "awaiting-submission", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "next_action": "Install a real client against the native host and import production metadata-only events.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "blocked_checks": [ { "label": "External events", @@ -785,14 +785,6 @@ "expected": ">0", "status": "blocked", "next_action": "Import at least one metadata-only event from a real client." - }, - { - "label": "Adoption sample", - "field": "adoption_sample_count", - "actual": 0, - "expected": ">0", - "status": "blocked", - "next_action": "Telemetry must include adoption outcome evidence." } ], "commands": [ @@ -1337,7 +1329,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1396,8 +1388,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -1922,12 +1913,12 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1943,14 +1934,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -2055,7 +2046,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "skill_atlas": { "skill_count": 12, @@ -2093,7 +2084,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -2130,7 +2121,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -2225,7 +2216,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -2242,14 +2233,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -2258,7 +2249,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -2365,7 +2356,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -2483,7 +2476,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2535,8 +2528,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -2555,7 +2548,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -2875,7 +2870,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2912,7 +2907,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2933,8 +2928,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2950,8 +2945,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -4598,7 +4593,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.96, + "duration_ms": 46.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4626,7 +4621,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.37, + "duration_ms": 45.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4649,7 +4644,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.25, + "duration_ms": 48.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4677,7 +4672,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.14, + "duration_ms": 43.77, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4700,7 +4695,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.63, + "duration_ms": 43.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4728,7 +4723,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.11, + "duration_ms": 40.08, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4751,7 +4746,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.98, + "duration_ms": 46.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4778,7 +4773,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.13, + "duration_ms": 43.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4801,7 +4796,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.48, + "duration_ms": 40.54, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4830,7 +4825,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.28, + "duration_ms": 40.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5556,22 +5551,35 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 64, + "sample": [ + " M evidence/world_class/templates/provider-holdout.intake.json", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -5587,29 +5595,30 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (6/13 pass, 7 blocked)" + "world-class source checks are not all accepted (7/13 pass, 6 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "ready": false, + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5617,7 +5626,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1" + "sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5691,7 +5700,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "ab66dfd1a1f2fca481752aa164271a8c4f419cab9653f45dd63d1c73011713bb" + "sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d" }, { "label": "blind_review", @@ -5726,21 +5735,21 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 118486, - "sha256": "ca53fbf5fa66f574d34603b160c3c3a264b015ba2722deb930b5accb94b52af8" + "sha256": "390ddd84dcfb27f7d1c6a3e6216c526ff1ec1ca86c11ea22dfdcdc0403e71cda" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 25199, - "sha256": "18c8a142c8108071bc453367c1fd3589b3368c098d5fc27f4b1d11c19f0bfb92" + "sha256": "dbd04851a3a6a14273a2e62823ee6d1d181d2e91b88c8c34d2a1b5ca284bc35a" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "7f980caf507c7d094401a18194eeeb7480de3a924326b6d0c97e07b33e3fb64f" + "sha256": "7762378b6fe7a3228b24e6a69499efb8e7ea3c394de1c9714c4d886fc38857ff" }, { "label": "package_verification", @@ -5753,78 +5762,78 @@ "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "aa45be12d2a94664dce95884668839d58808eaad70de90cfb1e96939e9e3a3c0" + "bytes": 8758, + "sha256": "ba0b8b088b64b1b60715075da4502048c50b2b00d922c6cc7bd3456dca148b34" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "a4dfc4e3af1167e3d6ced9444c41bc6cd2133bb67942fedba94c046814bf7866" + "sha256": "6187a8850680edd24e84161003d3d06379f5f9665ad1ed6e509b661283c62a44" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 20554, - "sha256": "38e95161404d2dc9a1c7bf3c1ee213a051ced5444deb01b0a01e557a4321021d" + "bytes": 20828, + "sha256": "af3c3f101ad992820bf36ddb14b02e7404ef16411567c483f86eed530ca45f62" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 21587, - "sha256": "d08019f31a7811ee747392f24a11902f173201054894c0338d80b45543c88d18" + "bytes": 21721, + "sha256": "6c135aced3443b88cc45009957386b566be80fd5c7f261361163dada861dab43" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 19178, - "sha256": "4a9119aba62b7cbc9f6b434c33f58a73d2fd623e1f4ed67c3ca994f23025a317" + "bytes": 19319, + "sha256": "69afd0ff3d49ba9e1ea57248e41cfe62a0abfdf06fd0f997ed3008a5656c2566" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 41071, - "sha256": "37fd4b91f92ceda3a5873431d5629b33c7909c10fbdc465eaad546e45562328f" + "bytes": 41202, + "sha256": "97924dd4be88bedd734bb0530765ce5ea6e10a92d2a17807c14b2aae5b66f960" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 13658, - "sha256": "6aaf868374ba134afbb4d4bf9c0339604ab41b7439379241359b83c279e9207f" + "bytes": 13655, + "sha256": "17def7ba6982d834e13ce3896dd9093019eec3ee8d2fa5b283ef8d6b17b19be0" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 24330, - "sha256": "40d1796964450e55bbaea98e46b0aa3433693e0b532c89b77ce5279c020570ff" + "bytes": 24403, + "sha256": "f2e53f857320f4c98faa8375e6b4758a8b6b291497578ec9adb14baeb6ee169c" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 15360, - "sha256": "ebfa3a4b528044bdf40f8931a246b54eb7cf37afe55bc02cad95f6e04ea4efa0" + "bytes": 15424, + "sha256": "337fb764b98ad2707f5a0d7bba88b9c5401256d7be00f8f3dc478e89e3bcb6c2" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 21284, - "sha256": "eb6c4506b0f25f01eef077c7c65bfe3f3230d3991b0db3befa8798768254373f" + "bytes": 21348, + "sha256": "ca72c65b437b2211d08b9d463c9a33a7993b4e388bb75f62757ead4dfcf81a49" }, { "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, "bytes": 18406, - "sha256": "704aba252d55e9b9f546392ce7f51b91f3b74d3384501d7f80a74e14c5b50acd" + "sha256": "25206d72ff919a2a4ce398b80a602028ab5f6a43648b5d3b690b3fdd02f49280" } ], "missing_artifacts": [], @@ -6775,7 +6784,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, @@ -11589,7 +11598,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -11629,7 +11638,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11726,7 +11735,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11808,7 +11817,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11890,7 +11899,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -12001,7 +12010,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "failures": [], "warnings": [], @@ -14967,7 +14976,7 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", @@ -16162,7 +16171,7 @@ "architecture_maintainability": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "python_file_count": 191, @@ -16190,6 +16199,13 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 874, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -16197,16 +16213,9 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 867, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", - "lines": 834, + "lines": 836, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -16276,6 +16285,13 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 874, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -16283,16 +16299,9 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 867, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "tests/verify_review_studio.py", - "lines": 834, + "lines": 836, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -16355,15 +16364,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1045126, + "other_text_tokens": 1046015, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1045923, - "deferred_resource_tokens": 484750, + "estimated_total_text_tokens": 1046812, + "deferred_resource_tokens": 484786, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122 }, { @@ -16385,7 +16394,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122 } ], @@ -16408,7 +16417,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424772, + "estimated_tokens": 424808, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -17259,7 +17268,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -17470,8 +17479,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -18134,7 +18143,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" @@ -18302,7 +18311,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-16T00:06:24Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -18326,14 +18335,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -18342,7 +18351,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -18354,31 +18363,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 0, - "accepted": 0, + "adoption_events": 1, + "accepted": 1, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 0 + "adoption_rate": 100.0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "review_event", + "event": "skill_activation", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "manual", - "outcome": "reviewed", + "activation_type": "explicit", + "outcome": "accepted", "failure_type": "none", - "timestamp": "2026-06-13T12:00:00Z" + "timestamp": "2026-06-13T10:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "reports/telemetry_events.jsonl", + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -18399,7 +18408,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -18416,7 +18425,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -18865,7 +18874,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ @@ -18914,7 +18923,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -18934,7 +18943,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -19186,7 +19195,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 @@ -19221,7 +19230,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -19271,7 +19280,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -19301,7 +19310,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -19736,8 +19745,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -19752,8 +19761,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -19797,7 +19806,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -20117,7 +20128,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -20154,7 +20165,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -20175,8 +20186,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -20192,8 +20203,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -20343,7 +20354,9 @@ "observed token metadata" ], "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -20540,7 +20553,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -20624,8 +20637,8 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "pending_count": 4, "ready_to_claim_world_class": false, "credential_value_exposed": false, @@ -20783,7 +20796,9 @@ "artifact_prefill_counts_as_evidence": false }, "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -21143,8 +21158,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -21499,8 +21514,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -21583,8 +21598,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -21596,8 +21611,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -21871,7 +21886,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -21892,8 +21907,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -21909,8 +21924,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -21947,8 +21962,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -21970,7 +21985,9 @@ "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 0; token-observed 0", "execution_runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -22332,7 +22349,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -22380,7 +22397,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22401,8 +22418,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -22417,10 +22434,9 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 2, + "blocked_source_check_count": 1, "next_source_actions": [ - "Import at least one metadata-only event from a real client.", - "Telemetry must include adoption outcome evidence." + "Import at least one metadata-only event from a real client." ], "submission_state": { "status": "missing", @@ -23248,7 +23264,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -23285,11 +23301,11 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -23305,7 +23321,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -23987,11 +24003,11 @@ "install_simulation": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, "archive_entry_count": 647, @@ -24012,7 +24028,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -24258,16 +24274,16 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -24341,7 +24357,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -24356,8 +24372,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 8dc11dbe..9ace2823 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -471,8 +471,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -997,12 +996,12 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1018,14 +1017,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1130,7 +1129,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "skill_atlas": { "skill_count": 12, @@ -1168,7 +1167,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -1205,7 +1204,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1300,7 +1299,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -1317,14 +1316,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1333,7 +1332,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -1440,7 +1439,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1558,7 +1559,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1610,8 +1611,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1630,7 +1631,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1950,7 +1953,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1987,7 +1990,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2008,8 +2011,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2025,8 +2028,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 76fadc1f..66f2b1d9 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/review_waivers.json b/reports/review_waivers.json index aa2558e7..07eeccbc 100644 --- a/reports/review_waivers.json +++ b/reports/review_waivers.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -52,7 +52,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { diff --git a/reports/review_waivers.md b/reports/review_waivers.md index 6c8d594c..dbc942d1 100644 --- a/reports/review_waivers.md +++ b/reports/review_waivers.md @@ -35,7 +35,7 @@ - waiver allowed: `true` - risk: review pending 5; model-executed 0; output failures 0 - evidence: `reports/output_review_adjudication.md` -- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md` +- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md` - world-class boundary: Does not count as provider, human, or public world-class completion evidence. #### Required Review diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json index 9c2ffd6e..59c9a1a4 100644 --- a/reports/runtime_permission_probes.json +++ b/reports/runtime_permission_probes.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -42,7 +42,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -139,7 +139,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -221,7 +221,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -303,7 +303,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index c05e9830..ce7f483e 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 0e3c4682..be4ab92f 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `213` -- Package SHA256: `db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336` +- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f` ## Failures diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index d04bb40e..efd1f1f6 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -919,7 +919,7 @@ -

世界证据World Evidence

世界级证据尚未完成:4 项待补,0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks6 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
+

世界证据World Evidence

世界级证据尚未完成:4 项待补,0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks7 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 564a097a..4f650692 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -471,8 +471,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -1001,12 +1000,12 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1022,14 +1021,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1134,7 +1133,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "skill_atlas": { "skill_count": 12, @@ -1172,7 +1171,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -1209,7 +1208,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1304,7 +1303,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -1321,14 +1320,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1337,7 +1336,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -1444,7 +1443,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1562,7 +1563,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1614,8 +1615,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1634,7 +1635,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1954,7 +1957,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1991,7 +1994,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2012,8 +2015,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2029,8 +2032,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill-overview.html b/reports/skill-overview.html index f999d092..d41e78e8 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -919,7 +919,7 @@ -

世界证据World Evidence

世界级证据尚未完成:4 项待补,0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks6 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
+

世界证据World Evidence

世界级证据尚未完成:4 项待补,0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks7 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index ce04180a..2dcc4e05 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -411,7 +411,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成:4 项待补,0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -470,8 +470,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -996,12 +995,12 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1", - "source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f", + "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1017,14 +1016,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 64 }, - "commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e", + "commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1129,7 +1128,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" }, "skill_atlas": { "skill_count": 12, @@ -1167,7 +1166,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336", + "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81" }, "compatibility": { @@ -1204,7 +1203,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1299,7 +1298,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -1316,14 +1315,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1332,7 +1331,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -1439,7 +1438,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1557,7 +1558,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1609,8 +1610,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1629,7 +1630,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1949,7 +1952,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1986,7 +1989,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2007,8 +2010,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2024,8 +2027,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 85769028..ae6eddde 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -44,7 +44,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -255,8 +255,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -919,7 +919,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 882c3f8d..8559a597 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -351,7 +351,7 @@ "key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "external_required", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "target": "A real Browser/Chrome/provider client sends production metadata events", "evidence": [ { @@ -433,7 +433,7 @@ "key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "external_required", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "target": "A real Browser/Chrome/provider client sends production metadata events", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 4bd96bde..1bf27b9b 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -30,7 +30,7 @@ Generated at: `2026-06-16` | Registry Distribution | pass | zip entries 647; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | -| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | +| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | ## Open Highest-Leverage Gaps diff --git a/reports/skillops/daily/2026-06-16.json b/reports/skillops/daily/2026-06-16.json index a35cbc88..386b0613 100644 --- a/reports/skillops/daily/2026-06-16.json +++ b/reports/skillops/daily/2026-06-16.json @@ -14,7 +14,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -31,7 +31,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -480,7 +480,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ diff --git a/reports/skillops/daily/2026-06-16.md b/reports/skillops/daily/2026-06-16.md index 105c41c3..d0fbea0f 100644 --- a/reports/skillops/daily/2026-06-16.md +++ b/reports/skillops/daily/2026-06-16.md @@ -14,7 +14,7 @@ Generated at: `2026-06-16` - local blueprint ready: `true` - public world-class ready: `false` - world-class pending: `4` -- release lock ready: `true` +- release lock ready: `false` - evidence consistency ok: `true` This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json index 998d23ea..65c79eb3 100644 --- a/reports/skillops/weekly/2026-W25.json +++ b/reports/skillops/weekly/2026-W25.json @@ -15,7 +15,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -35,7 +35,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -287,7 +287,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md index 4d75fc31..1fd6a716 100644 --- a/reports/skillops/weekly/2026-W25.md +++ b/reports/skillops/weekly/2026-W25.md @@ -13,7 +13,7 @@ Week: `2026-W25` - top_score: `88` - skill_count: `12` - actionable_portfolio_issue_count: `0` -- release_lock_ready: `True` +- release_lock_ready: `False` - evidence_consistency_ok: `True` - public_world_class_ready: `False` - world_class_pending_count: `4` diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index b07bb792..ee01af92 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -75,7 +75,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336" + "to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" } ] }, @@ -90,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } } diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index d42e2d05..f97f646f 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -113,7 +113,9 @@ "observed token metadata" ], "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -310,7 +312,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index bdc44eb2..c5a01fae 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -83,7 +83,9 @@ This report validates the intake contract for human and external evidence. A val #### Source Runbook -- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` +- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value. +- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}` +- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. - `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index fc725830..3d24c166 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -15,8 +15,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -31,8 +31,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -76,7 +76,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -396,7 +398,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -433,7 +435,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -454,8 +456,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -471,8 +473,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 7fd7dc45..420fbb52 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -8,8 +8,8 @@ Generated at: `2026-06-16` - ready to claim world-class: `false` - entries: `4` - source accepted: `0` -- source checks: `6` pass / `13` total -- source blocked: `7` +- source checks: `7` pass / `13` total +- source blocked: `6` - accepted: `0` - pending: `4` - human pending: `1` @@ -29,7 +29,7 @@ This ledger records the current evidence state. It requires both passing source | `provider-holdout` | `pending` | `missing` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices in the decision template, then regenerate adjudication. | | `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | -| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | +| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. | ## Provider Holdout @@ -47,7 +47,9 @@ This ledger records the current evidence state. It requires both passing source ### Source Runbook -- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` +- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value. +- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}` +- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. - `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` @@ -169,8 +171,8 @@ This ledger records the current evidence state. It requires both passing source - objective: Import production metadata-only events from a real external client into the local drift loop. - source status: `external_required` -- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` -- source checks: `1` pass / `3` total +- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` +- source checks: `2` pass / `3` total - submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}` ### Provenance Requirements @@ -194,7 +196,7 @@ This ledger records the current evidence state. It requires both passing source | Check | Current | Expected | Status | | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | -| Adoption sample | `0` | `>0` | `blocked` | +| Adoption sample | `1` | `>0` | `pass` | | Raw content blocked | `False` | `false` | `pass` | ### Completion Assertions diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index dbe36747..845c0ad8 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -26,7 +26,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -144,7 +146,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -187,7 +189,9 @@ "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -305,7 +309,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index f2da2daa..8c5c9537 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It | `provider-holdout` | `external_required` | `external` | operator with provider credentials | model-executed 0; token-observed 0 | | `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 | | `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 | -| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 | +| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 | ## Provider Holdout @@ -31,7 +31,9 @@ This report is an execution plan for the remaining world-class evidence gaps. It ### Runbook -- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` +- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value. +- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}` +- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. - `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index f40d59e8..88f28e2d 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -65,7 +65,7 @@ Evidence Collection

World-Class Evidence Preflight

This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

-
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source6/13
+
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source7/13
@@ -234,7 +234,7 @@

Runbook

-
  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
@@ -611,15 +611,15 @@ -
+
adoption_sample_count Adoption sample
-
Current
0
+
Current
1
Expected
>0
-
Status
blocked
+
Status
pass
Action
Telemetry must include adoption outcome evidence.
diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index ecdf483a..615577db 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -14,8 +14,8 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "pending_count": 4, "ready_to_claim_world_class": false, "credential_value_exposed": false, @@ -173,7 +173,9 @@ "artifact_prefill_counts_as_evidence": false }, "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -533,8 +535,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -889,8 +891,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index 696f261d..b2349713 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -10,7 +10,7 @@ Generated at: `2026-06-16` - credential value exposed: `false` - collection ready: `1` - collection blocked: `3` -- source checks: `6` pass / `13` total +- source checks: `7` pass / `13` total This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger. @@ -150,7 +150,7 @@ Generate the submission kit after the real provider, human, native-permission, o | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 81f44874..9fa92ced 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -57,7 +57,7 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting4
Ready0
Source6/13
Blocked7
Invalid0
+
Pending4
Awaiting4
Ready0
Source7/13
Blocked6
Invalid0

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Evidence Items

@@ -69,7 +69,7 @@
Blocked
2
Submission
evidence/world_class/submissions/provider-holdout.json
-

Source Runbook

  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Source Runbook

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • provider-backed model run
  • observed timing
  • observed token metadata
@@ -120,7 +120,7 @@
Owner
Browser/Chrome/IDE/provider client integrator
Ledger
pending
-
Blocked
2
+
Blocked
1
Submission
evidence/world_class/submissions/native-client-telemetry.json

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
@@ -130,8 +130,8 @@

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
-

Next Source Actions

  • Import at least one metadata-only event from a real client.
  • Telemetry must include adoption outcome evidence.
-

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+

Next Source Actions

  • Import at least one metadata-only event from a real client.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

Boundary

  • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
  • Valid intake means ready for submission review; ledger review still requires passing source evidence.
  • The world-class ledger and claim guard remain the source of truth.
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index c889b801..bc72296d 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -12,8 +12,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -35,7 +35,9 @@ "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 0; token-observed 0", "execution_runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -397,7 +399,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -445,7 +447,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -466,8 +468,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -482,10 +484,9 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 2, + "blocked_source_check_count": 1, "next_source_actions": [ - "Import at least one metadata-only event from a real client.", - "Telemetry must include adoption outcome evidence." + "Import at least one metadata-only event from a real client." ], "submission_state": { "status": "missing", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 7d396272..9db0580c 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -28,7 +28,7 @@ This runbook coordinates evidence collection only. It does not accept submission | `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials. | operator with provider credentials | | `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Record a reviewer choice for every pair. | human reviewer | | `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | -| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | ## Provider Holdout @@ -40,7 +40,9 @@ This runbook coordinates evidence collection only. It does not accept submission ### Source Runbook -- YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 +- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value. +- export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} +- python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 - python3 scripts/yao.py skill-os2-audit . --generated-at - Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. - python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions @@ -242,7 +244,7 @@ This runbook coordinates evidence collection only. It does not accept submission - objective: Import production metadata-only events from a real external client into the local drift loop. - blocking reason: No evidence packet has been submitted for review. -- blocked source checks: `2` +- blocked source checks: `1` - submission: `evidence/world_class/submissions/native-client-telemetry.json` - template: `evidence/world_class/templates/native-client-telemetry.intake.json` @@ -295,14 +297,13 @@ This runbook coordinates evidence collection only. It does not accept submission ### Next Source Actions - Import at least one metadata-only event from a real client. -- Telemetry must include adoption outcome evidence. ### Source Evidence Snapshot | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 43a75d25..8eaeeee6 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -14,8 +14,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -27,8 +27,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -302,7 +302,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -323,8 +323,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -340,8 +340,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 7d625fcc..322fc7ff 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -139,7 +139,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks - External events: 0 / >0 => blocked -- Adoption sample: 0 / >0 => blocked +- Adoption sample: 1 / >0 => pass - Raw content blocked: False / false => pass #### Completion Assertions diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py index eb0b7c47..d0ec028a 100644 --- a/scripts/render_world_class_evidence_plan.py +++ b/scripts/render_world_class_evidence_plan.py @@ -17,7 +17,9 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { "owner": "operator with provider credentials", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", + "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", ], "success_checks": [ diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index aa7cbeaa..14b36f13 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -1,6 +1,6 @@ { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -211,8 +211,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, diff --git a/skill_atlas/stale_skills.json b/skill_atlas/stale_skills.json index b57d361d..6602e829 100644 --- a/skill_atlas/stale_skills.json +++ b/skill_atlas/stale_skills.json @@ -24,7 +24,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 357aa1bc..f43c9b9a 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -545,6 +545,7 @@ def main() -> None: assert any("world-class-intake" in item["command"] for item in provider_action_step["commands"]), provider_action_step assert any("world-class-ledger" in item["command"] for item in provider_action_step["commands"]), provider_action_step assert any("output-exec --provider-runner openai" in item for item in provider_action_step["runbook"]), provider_action_step + assert not any("" in item or "OPENAI_API_KEY=" in item for item in provider_action_step["runbook"]), provider_action_step assert "provider-backed model run" in provider_action_step["provenance_requirements"], provider_action_step assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider_action_step["success_checks"], provider_action_step assert "reports/output_execution_runs.json" in provider_action_step["evidence_artifacts"], provider_action_step @@ -727,6 +728,7 @@ def main() -> None: assert "计划、metadata fallback、待评审和本地命令不会被当成完成证据" in html, html assert "执行步骤" in html, html assert "output-exec --provider-runner openai" in html, html + assert "<redacted>" not in html and "" not in html, html assert "world-runbook-panel" in html, html assert "源证据检查" in html, html assert "world-source-checks" in html, html diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 4ef7a785..269b82bb 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -604,6 +604,8 @@ def main() -> None: assert "`python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`" in markdown, markdown assert "Source Runbook" in markdown, markdown assert "output-exec --provider-runner openai" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown assert "Templates and planned work do not count as accepted evidence." in markdown, markdown assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown @@ -669,6 +671,9 @@ def main() -> None: assert kit_draft["template_only"] is True, kit_draft assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft + assert kit_draft["provenance"]["run_command"] == "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", kit_draft + assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft + assert "" not in json.dumps(kit_draft), kit_draft kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"] @@ -680,6 +685,7 @@ def main() -> None: assert "Drafts are not accepted evidence." in kit_readme, kit_readme assert "Execution Runbook" in kit_readme, kit_readme assert "output-exec --provider-runner openai" in kit_readme, kit_readme + assert "" not in kit_readme, kit_readme assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme assert "Evidence Matrix" in kit_readme, kit_readme @@ -715,6 +721,7 @@ def main() -> None: assert "World-Class Evidence Submission Kit" in kit_html, kit_html assert "Execution Runbook" in kit_html, kit_html assert "output-exec --provider-runner openai" in kit_html, kit_html + assert "<redacted>" not in kit_html and "" not in kit_html, kit_html assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py index fa00b3ac..3f7357be 100644 --- a/tests/verify_world_class_evidence_ledger.py +++ b/tests/verify_world_class_evidence_ledger.py @@ -122,6 +122,7 @@ def main() -> None: }, entries assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"] assert any("output-exec --provider-runner openai" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] + assert not any("" in step or "OPENAI_API_KEY=" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]} assert provider_source["model_executed_count"]["status"] == "blocked", provider_source assert provider_source["timing_observed_count"]["status"] == "pass", provider_source @@ -147,6 +148,8 @@ def main() -> None: assert "source checks:" in markdown, markdown assert "Source Runbook" in markdown, markdown assert "output-exec --provider-runner openai" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown assert "Source Evidence Checks" in markdown, markdown assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown assert "`provider-holdout`" in markdown, markdown diff --git a/tests/verify_world_class_evidence_plan.py b/tests/verify_world_class_evidence_plan.py index 0a72b6d0..3f8393ec 100644 --- a/tests/verify_world_class_evidence_plan.py +++ b/tests/verify_world_class_evidence_plan.py @@ -54,7 +54,11 @@ def main() -> None: "native-client-telemetry", }, tasks assert set(requirements) == set(tasks), requirements - assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] + provider_runbook = tasks["provider-holdout"]["runbook"] + assert any("--provider-runner openai" in command for command in provider_runbook), tasks["provider-holdout"] + assert any(command.startswith("Set OPENAI_API_KEY") for command in provider_runbook), tasks["provider-holdout"] + assert any(command.startswith("export YAO_OUTPUT_EVAL_MODEL=") for command in provider_runbook), tasks["provider-holdout"] + assert not any("" in command or "OPENAI_API_KEY=" in command for command in provider_runbook), tasks["provider-holdout"] assert any("world-class-intake . --submissions-dir evidence/world_class/submissions" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert "reports/output_review_decisions.json" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"] @@ -76,6 +80,8 @@ def main() -> None: assert "`provider-holdout`" in markdown, markdown assert "ready to claim world-class: `false`" in markdown, markdown assert "ledger completion required: `true`" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py index c4edbae9..c04738d7 100644 --- a/tests/verify_world_class_operator_runbook.py +++ b/tests/verify_world_class_operator_runbook.py @@ -122,6 +122,7 @@ def main() -> None: assert provider["source_accepted"] is False, provider assert provider["blocked_source_check_count"] == 2, provider assert any("output-exec --provider-runner openai" in step for step in provider["execution_runbook"]), provider + assert not any("" in step or "OPENAI_API_KEY=" in step for step in provider["execution_runbook"]), provider assert "Run provider-backed output-exec with real credentials." in provider["next_source_actions"], provider assert "Provider execution should return non-estimated token usage." in provider["next_source_actions"], provider assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider @@ -143,6 +144,8 @@ def main() -> None: assert "| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials." in markdown, markdown assert "Source Runbook" in markdown, markdown assert "output-exec --provider-runner openai" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown assert "### Next Source Actions" in markdown, markdown assert "- Provider execution should return non-estimated token usage." in markdown, markdown assert "Source Evidence Snapshot" in markdown, markdown @@ -159,6 +162,8 @@ def main() -> None: assert "Next Source Actions" in html, html assert "Source Runbook" in html, html assert "output-exec --provider-runner openai" in html, html + assert "<redacted>" not in html and "" not in html, html + assert "OPENAI_API_KEY=<redacted>" not in html, html assert "Source Evidence Snapshot" in html, html assert "model_executed_count" in html, html assert "model_executed_count: 0 / >0" in html, html