Make provider evidence runbook copy safe
This commit is contained in:
yaojingang
@@ -18,7 +18,10 @@
|
||||
"provenance": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4.1-mini",
|
||||
"run_command": "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"run_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"credential_env": "OPENAI_API_KEY",
|
||||
"model_env": "YAO_OUTPUT_EVAL_MODEL",
|
||||
"model_default": "gpt-4.1-mini",
|
||||
"credential_material_committed": false
|
||||
},
|
||||
"privacy": {
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"package_count": 1,
|
||||
"packages": [
|
||||
{
|
||||
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -53,5 +53,5 @@
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-16"
|
||||
"generated_at": "2026-06-13"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-16T00:06:24Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
@@ -25,14 +25,14 @@
|
||||
},
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -41,7 +41,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -53,31 +53,31 @@
|
||||
{
|
||||
"skill": "yao-meta-skill",
|
||||
"events": 1,
|
||||
"adoption_events": 0,
|
||||
"accepted": 0,
|
||||
"adoption_events": 1,
|
||||
"accepted": 1,
|
||||
"edited": 0,
|
||||
"rejected": 0,
|
||||
"missed": 0,
|
||||
"adoption_rate": 0
|
||||
"adoption_rate": 100.0
|
||||
}
|
||||
],
|
||||
"next_iteration_candidates": [],
|
||||
"recent_events": [
|
||||
{
|
||||
"command": "unknown",
|
||||
"event": "review_event",
|
||||
"event": "skill_activation",
|
||||
"skill": "yao-meta-skill",
|
||||
"source": "manual",
|
||||
"version": "1.1.0",
|
||||
"activation_type": "manual",
|
||||
"outcome": "reviewed",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T12:00:00Z"
|
||||
"timestamp": "2026-06-13T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"failures": [],
|
||||
"artifacts": {
|
||||
"events_jsonl": "reports/telemetry_events.jsonl",
|
||||
"events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl",
|
||||
"json": "reports/adoption_drift_report.json",
|
||||
"markdown": "reports/adoption_drift_report.md"
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
## Summary
|
||||
|
||||
- Events: `1`
|
||||
- Adoption samples: `0`
|
||||
- Activation events: `0`
|
||||
- Adoption rate: `0`
|
||||
- Adoption samples: `1`
|
||||
- Activation events: `1`
|
||||
- Adoption rate: `100.0`
|
||||
- Missed trigger signals: `0`
|
||||
- Bad output signals: `0`
|
||||
- Script error signals: `0`
|
||||
@@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
|
||||
| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 |
|
||||
|
||||
## Next Iteration Candidates
|
||||
|
||||
@@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
|
||||
## Recent Metadata Events
|
||||
|
||||
- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none`
|
||||
- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"python_file_count": 191,
|
||||
@@ -29,6 +29,13 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_world_class_evidence_intake.py",
|
||||
"lines": 874,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_parser.py",
|
||||
"lines": 870,
|
||||
@@ -36,16 +43,9 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_world_class_evidence_intake.py",
|
||||
"lines": 867,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_review_studio.py",
|
||||
"lines": 834,
|
||||
"lines": 836,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
@@ -115,6 +115,13 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_world_class_evidence_intake.py",
|
||||
"lines": 874,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_parser.py",
|
||||
"lines": 870,
|
||||
@@ -122,16 +129,9 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_world_class_evidence_intake.py",
|
||||
"lines": 867,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_review_studio.py",
|
||||
"lines": 834,
|
||||
"lines": 836,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Architecture Maintainability
|
||||
|
||||
Generated at: `2026-06-16`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
@@ -30,9 +30,9 @@ No file-size hotspots found.
|
||||
| File | Lines | Kind | Recommended next split |
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_review_studio.py` | `834` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_review_studio.py` | `836` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `scripts/render_world_class_preflight.py` | `814` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
|
||||
| `scripts/render_evidence_consistency.py` | `808` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
|
||||
| `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
|
||||
@@ -45,9 +45,9 @@ No file-size hotspots found.
|
||||
| File | Lines | Kind | Severity |
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_yao_cli.py` | `899` | `test` | `pass` |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | `pass` |
|
||||
| `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `867` | `test` | `pass` |
|
||||
| `tests/verify_review_studio.py` | `834` | `test` | `pass` |
|
||||
| `tests/verify_review_studio.py` | `836` | `test` | `pass` |
|
||||
| `scripts/render_world_class_preflight.py` | `814` | `cli-script` | `pass` |
|
||||
| `scripts/render_evidence_consistency.py` | `808` | `cli-script` | `pass` |
|
||||
| `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` |
|
||||
|
||||
@@ -3,22 +3,35 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"skill_dir": ".",
|
||||
"commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"sample": [],
|
||||
"dirty": true,
|
||||
"changed_file_count": 64,
|
||||
"sample": [
|
||||
" M evidence/world_class/templates/provider-holdout.intake.json",
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/adoption_drift_report.md",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/compiled_targets.json",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/context_budget.md"
|
||||
],
|
||||
"scope": "generation-time status before this report is written"
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1",
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -34,29 +47,30 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 64
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": false,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": [
|
||||
"release lock is not clean or commit is unavailable",
|
||||
"provider-backed model holdout evidence is incomplete",
|
||||
"human blind-review adjudication is incomplete",
|
||||
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
|
||||
"world-class source checks are not all accepted (6/13 pass, 7 blocked)"
|
||||
"world-class source checks are not all accepted (7/13 pass, 6 blocked)"
|
||||
],
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"ready": false,
|
||||
"commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"reason": "clean generation-time HEAD"
|
||||
"reason": "working tree was dirty at generation time"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
@@ -64,7 +78,7 @@
|
||||
"existing_count": 25,
|
||||
"missing_count": 0,
|
||||
"missing_paths": [],
|
||||
"sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1"
|
||||
"sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f"
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -138,7 +152,7 @@
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "ab66dfd1a1f2fca481752aa164271a8c4f419cab9653f45dd63d1c73011713bb"
|
||||
"sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -173,21 +187,21 @@
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 118486,
|
||||
"sha256": "ca53fbf5fa66f574d34603b160c3c3a264b015ba2722deb930b5accb94b52af8"
|
||||
"sha256": "390ddd84dcfb27f7d1c6a3e6216c526ff1ec1ca86c11ea22dfdcdc0403e71cda"
|
||||
},
|
||||
{
|
||||
"label": "python_compatibility",
|
||||
"path": "reports/python_compatibility.json",
|
||||
"exists": true,
|
||||
"bytes": 25199,
|
||||
"sha256": "18c8a142c8108071bc453367c1fd3589b3368c098d5fc27f4b1d11c19f0bfb92"
|
||||
"sha256": "dbd04851a3a6a14273a2e62823ee6d1d181d2e91b88c8c34d2a1b5ca284bc35a"
|
||||
},
|
||||
{
|
||||
"label": "registry_audit",
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "7f980caf507c7d094401a18194eeeb7480de3a924326b6d0c97e07b33e3fb64f"
|
||||
"sha256": "7762378b6fe7a3228b24e6a69499efb8e7ea3c394de1c9714c4d886fc38857ff"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
@@ -200,78 +214,78 @@
|
||||
"label": "install_simulation",
|
||||
"path": "reports/install_simulation.json",
|
||||
"exists": true,
|
||||
"bytes": 8604,
|
||||
"sha256": "aa45be12d2a94664dce95884668839d58808eaad70de90cfb1e96939e9e3a3c0"
|
||||
"bytes": 8758,
|
||||
"sha256": "ba0b8b088b64b1b60715075da4502048c50b2b00d922c6cc7bd3456dca148b34"
|
||||
},
|
||||
{
|
||||
"label": "skill_os2_audit",
|
||||
"path": "reports/skill_os2_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 14310,
|
||||
"sha256": "a4dfc4e3af1167e3d6ced9444c41bc6cd2133bb67942fedba94c046814bf7866"
|
||||
"sha256": "6187a8850680edd24e84161003d3d06379f5f9665ad1ed6e509b661283c62a44"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_plan",
|
||||
"path": "reports/world_class_evidence_plan.json",
|
||||
"exists": true,
|
||||
"bytes": 20554,
|
||||
"sha256": "38e95161404d2dc9a1c7bf3c1ee213a051ced5444deb01b0a01e557a4321021d"
|
||||
"bytes": 20828,
|
||||
"sha256": "af3c3f101ad992820bf36ddb14b02e7404ef16411567c483f86eed530ca45f62"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_ledger",
|
||||
"path": "reports/world_class_evidence_ledger.json",
|
||||
"exists": true,
|
||||
"bytes": 21587,
|
||||
"sha256": "d08019f31a7811ee747392f24a11902f173201054894c0338d80b45543c88d18"
|
||||
"bytes": 21721,
|
||||
"sha256": "6c135aced3443b88cc45009957386b566be80fd5c7f261361163dada861dab43"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_intake",
|
||||
"path": "reports/world_class_evidence_intake.json",
|
||||
"exists": true,
|
||||
"bytes": 19178,
|
||||
"sha256": "4a9119aba62b7cbc9f6b434c33f58a73d2fd623e1f4ed67c3ca994f23025a317"
|
||||
"bytes": 19319,
|
||||
"sha256": "69afd0ff3d49ba9e1ea57248e41cfe62a0abfdf06fd0f997ed3008a5656c2566"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_preflight",
|
||||
"path": "reports/world_class_evidence_preflight.json",
|
||||
"exists": true,
|
||||
"bytes": 41071,
|
||||
"sha256": "37fd4b91f92ceda3a5873431d5629b33c7909c10fbdc465eaad546e45562328f"
|
||||
"bytes": 41202,
|
||||
"sha256": "97924dd4be88bedd734bb0530765ce5ea6e10a92d2a17807c14b2aae5b66f960"
|
||||
},
|
||||
{
|
||||
"label": "world_class_submission_review",
|
||||
"path": "reports/world_class_submission_review.json",
|
||||
"exists": true,
|
||||
"bytes": 13658,
|
||||
"sha256": "6aaf868374ba134afbb4d4bf9c0339604ab41b7439379241359b83c279e9207f"
|
||||
"bytes": 13655,
|
||||
"sha256": "17def7ba6982d834e13ce3896dd9093019eec3ee8d2fa5b283ef8d6b17b19be0"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook",
|
||||
"path": "reports/world_class_operator_runbook.json",
|
||||
"exists": true,
|
||||
"bytes": 24330,
|
||||
"sha256": "40d1796964450e55bbaea98e46b0aa3433693e0b532c89b77ce5279c020570ff"
|
||||
"bytes": 24403,
|
||||
"sha256": "f2e53f857320f4c98faa8375e6b4758a8b6b291497578ec9adb14baeb6ee169c"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_markdown",
|
||||
"path": "reports/world_class_operator_runbook.md",
|
||||
"exists": true,
|
||||
"bytes": 15360,
|
||||
"sha256": "ebfa3a4b528044bdf40f8931a246b54eb7cf37afe55bc02cad95f6e04ea4efa0"
|
||||
"bytes": 15424,
|
||||
"sha256": "337fb764b98ad2707f5a0d7bba88b9c5401256d7be00f8f3dc478e89e3bcb6c2"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_html",
|
||||
"path": "reports/world_class_operator_runbook.html",
|
||||
"exists": true,
|
||||
"bytes": 21284,
|
||||
"sha256": "eb6c4506b0f25f01eef077c7c65bfe3f3230d3991b0db3befa8798768254373f"
|
||||
"bytes": 21348,
|
||||
"sha256": "ca72c65b437b2211d08b9d463c9a33a7993b4e388bb75f62757ead4dfcf81a49"
|
||||
},
|
||||
{
|
||||
"label": "world_class_claim_guard",
|
||||
"path": "reports/world_class_claim_guard.json",
|
||||
"exists": true,
|
||||
"bytes": 18406,
|
||||
"sha256": "704aba252d55e9b9f546392ce7f51b91f3b74d3384501d7f80a74e14c5b50acd"
|
||||
"sha256": "25206d72ff919a2a4ce398b80a602028ab5f6a43648b5d3b690b3fdd02f49280"
|
||||
}
|
||||
],
|
||||
"missing_artifacts": [],
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-16`
|
||||
Commit: `4b2767900bbbbb962d2b11a92610951c3a0a0f0e`
|
||||
Working tree dirty at generation: `false`
|
||||
Evidence bundle SHA256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1`
|
||||
Commit: `54851019f53f3ee3f508dedfc0daaf192d6bc6ad`
|
||||
Working tree dirty at generation: `true`
|
||||
Evidence bundle SHA256: `95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- release lock ready: `true`
|
||||
- release lock ready: `false`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `25`
|
||||
- missing artifacts: `0`
|
||||
- source contract sha256: `db36e8f9d7d6`
|
||||
- source contract sha256: `a5f307ba8e89`
|
||||
- archive sha256: `996e36863203`
|
||||
- output cases: `5`
|
||||
- disclosed failure cases: `3`
|
||||
@@ -20,10 +20,10 @@ Evidence bundle SHA256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- world-class source checks: `6` pass / `13` total; `7` blocked
|
||||
- world-class source checks: `7` pass / `13` total; `6` blocked
|
||||
- public claim ready: `false`
|
||||
- public claim blockers: `4`
|
||||
- changed files at generation: `0`
|
||||
- public claim blockers: `5`
|
||||
- changed files at generation: `64`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
|
||||
|
||||
@@ -35,22 +35,23 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
| Blocker |
|
||||
| --- |
|
||||
| release lock is not clean or commit is unavailable |
|
||||
| provider-backed model holdout evidence is incomplete |
|
||||
| human blind-review adjudication is incomplete |
|
||||
| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) |
|
||||
| world-class source checks are not all accepted (6/13 pass, 7 blocked) |
|
||||
| world-class source checks are not all accepted (7/13 pass, 6 blocked) |
|
||||
|
||||
## Release Lock
|
||||
|
||||
- ready: `true`
|
||||
- reason: clean generation-time HEAD
|
||||
- ready: `false`
|
||||
- reason: working tree was dirty at generation time
|
||||
- status scope: generation-time status before this report is written
|
||||
|
||||
## Evidence Bundle
|
||||
|
||||
- algorithm: `sha256(path,label,exists,artifact_sha256)`
|
||||
- artifacts: `25` / `25`
|
||||
- sha256: `7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1`
|
||||
- sha256: `95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f`
|
||||
|
||||
## Methodology Sections
|
||||
|
||||
@@ -72,26 +73,26 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `ab66dfd1a1f2` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `072e7ff6ef9c` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `ca53fbf5fa66` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `18c8a142c810` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `7f980caf507c` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `390ddd84dcfb` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `dbd04851a3a6` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `7762378b6fe7` |
|
||||
| package_verification | `reports/package_verification.json` | present | `bbe73152879c` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `aa45be12d2a9` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4dfc4e3af11` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `38e95161404d` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `d08019f31a78` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `4a9119aba62b` |
|
||||
| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `37fd4b91f92c` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `6aaf868374ba` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `40d179696445` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `ebfa3a4b5280` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `eb6c4506b0f2` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `704aba252d55` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `ba0b8b088b64` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `6187a8850680` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `af3c3f101ad9` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `6c135aced344` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `69afd0ff3d49` |
|
||||
| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `97924dd4be88` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `17def7ba6982` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `f2e53f857320` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `337fb764b98a` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `ca72c65b437b` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `25206d72ff91` |
|
||||
|
||||
## Reproduction Commands
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"target_count": 5,
|
||||
|
||||
@@ -6,15 +6,15 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 797,
|
||||
"other_text_tokens": 1045126,
|
||||
"other_text_tokens": 1046015,
|
||||
"estimated_initial_load_tokens": 990,
|
||||
"estimated_total_text_tokens": 1045923,
|
||||
"deferred_resource_tokens": 484750,
|
||||
"estimated_total_text_tokens": 1046812,
|
||||
"deferred_resource_tokens": 484786,
|
||||
"deferred_resource_warn_threshold": 120000,
|
||||
"deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 424772,
|
||||
"estimated_tokens": 424808,
|
||||
"file_count": 122
|
||||
},
|
||||
{
|
||||
@@ -36,7 +36,7 @@
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 424772,
|
||||
"estimated_tokens": 424808,
|
||||
"file_count": 122
|
||||
}
|
||||
],
|
||||
@@ -59,7 +59,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 424772,
|
||||
"estimated_tokens": 424808,
|
||||
"file_count": 122,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 484750 | `governed` | scripts:424772 | 131.3 | - | ok |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 484786 | `governed` | scripts:424808 | 131.3 | - | ok |
|
||||
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
|
||||
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
"budget_limit": 1000,
|
||||
"initial_tokens": 990,
|
||||
"skill_body_tokens": 797,
|
||||
"deferred_resource_tokens": 484750,
|
||||
"deferred_resource_tokens": 484786,
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 424772,
|
||||
"estimated_tokens": 424808,
|
||||
"file_count": 122
|
||||
}
|
||||
],
|
||||
@@ -35,7 +35,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 424772,
|
||||
"estimated_tokens": 424808,
|
||||
"file_count": 122,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -189,12 +189,12 @@
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"status": "pass",
|
||||
"detail": "initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3",
|
||||
"detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3",
|
||||
"evidence": "reports/context_budget.json"
|
||||
},
|
||||
"actual": {
|
||||
"status": "pass",
|
||||
"detail": "initial load 990/1000; deferred 484750/120000; top deferred scripts 424772; resource governance governed; quality density 131.3",
|
||||
"detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3",
|
||||
"evidence": "reports/context_budget.json"
|
||||
},
|
||||
"paths": [
|
||||
@@ -207,8 +207,8 @@
|
||||
"key": "benchmark-release-lock-self-consistency",
|
||||
"label": "Benchmark release lock matches git dirty state",
|
||||
"status": "pass",
|
||||
"expected": true,
|
||||
"actual": true,
|
||||
"expected": false,
|
||||
"actual": false,
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json"
|
||||
],
|
||||
@@ -248,8 +248,8 @@
|
||||
"key": "overview-benchmark-commit",
|
||||
"label": "overview embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"actual": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"expected": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"actual": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-overview.json"
|
||||
@@ -261,30 +261,30 @@
|
||||
"label": "overview embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -298,13 +298,13 @@
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"source_types": {
|
||||
"manual": 1
|
||||
@@ -312,13 +312,13 @@
|
||||
},
|
||||
"actual": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"source_types": {
|
||||
"manual": 1
|
||||
@@ -341,8 +341,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
},
|
||||
@@ -353,8 +353,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
},
|
||||
@@ -374,7 +374,7 @@
|
||||
"pending_count": 4,
|
||||
"accepted_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6
|
||||
"source_pass_count": 7
|
||||
},
|
||||
"actual": {
|
||||
"ready": false,
|
||||
@@ -382,7 +382,7 @@
|
||||
"pending_count": 4,
|
||||
"accepted_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6
|
||||
"source_pass_count": 7
|
||||
},
|
||||
"paths": [
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
@@ -394,8 +394,8 @@
|
||||
"key": "interpretation-benchmark-commit",
|
||||
"label": "interpretation embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"actual": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"expected": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"actual": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-interpretation.json"
|
||||
@@ -407,30 +407,30 @@
|
||||
"label": "interpretation embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -444,13 +444,13 @@
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"source_types": {
|
||||
"manual": 1
|
||||
@@ -458,13 +458,13 @@
|
||||
},
|
||||
"actual": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"source_types": {
|
||||
"manual": 1
|
||||
@@ -487,8 +487,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
},
|
||||
@@ -499,8 +499,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
},
|
||||
@@ -520,7 +520,7 @@
|
||||
"pending_count": 4,
|
||||
"accepted_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6
|
||||
"source_pass_count": 7
|
||||
},
|
||||
"actual": {
|
||||
"ready": false,
|
||||
@@ -528,7 +528,7 @@
|
||||
"pending_count": 4,
|
||||
"accepted_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6
|
||||
"source_pass_count": 7
|
||||
},
|
||||
"paths": [
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
@@ -1327,7 +1327,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_pass_count": 7,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -1386,8 +1386,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1401,7 +1400,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_pass_count": 7,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -1460,8 +1459,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1811,15 +1809,15 @@
|
||||
"expected": {
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false
|
||||
},
|
||||
"actual": {
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false
|
||||
},
|
||||
"paths": [
|
||||
@@ -1835,8 +1833,8 @@
|
||||
"expected": {
|
||||
"pending_count": 4,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"preflight_counts_as_evidence": false,
|
||||
"credential_value_exposed": false
|
||||
@@ -1844,8 +1842,8 @@
|
||||
"actual": {
|
||||
"pending_count": 4,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"preflight_counts_as_evidence": false,
|
||||
"credential_value_exposed": false
|
||||
@@ -2099,8 +2097,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"plan_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
@@ -2248,8 +2246,8 @@
|
||||
"human_pending_count": 1,
|
||||
"external_pending_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"plan_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"package_dir": "dist",
|
||||
"install_root": "dist/install-simulation/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"package_dir": "tests/tmp_review_studio/dist",
|
||||
"install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 647,
|
||||
@@ -26,7 +26,7 @@
|
||||
{
|
||||
"id": "archive-present",
|
||||
"status": "pass",
|
||||
"detail": "Package archive exists: dist/yao-meta-skill.zip"
|
||||
"detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip"
|
||||
},
|
||||
{
|
||||
"id": "archive-safe-paths",
|
||||
@@ -272,9 +272,9 @@
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
"artifacts": {
|
||||
"archive": "dist/yao-meta-skill.zip",
|
||||
"package_manifest": "dist/manifest.json",
|
||||
"json": "reports/install_simulation.json",
|
||||
"markdown": "reports/install_simulation.md"
|
||||
"archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip",
|
||||
"package_manifest": "tests/tmp_review_studio/dist/manifest.json",
|
||||
"json": "tests/tmp_review_studio/install_simulation.json",
|
||||
"markdown": "tests/tmp_review_studio/install_simulation.md"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Install Simulation
|
||||
|
||||
- OK: `True`
|
||||
- Package directory: `dist`
|
||||
- Package directory: `tests/tmp_review_studio/dist`
|
||||
- Archive extracted: `True`
|
||||
- Entrypoint loaded: `True`
|
||||
- Manifest loaded: `True`
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
| Check | Status | Detail |
|
||||
| --- | --- | --- |
|
||||
| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip |
|
||||
| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip |
|
||||
| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries |
|
||||
| `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill |
|
||||
| `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable |
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 35.96,
|
||||
"duration_ms": 46.38,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 34.37,
|
||||
"duration_ms": 45.36,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 35.25,
|
||||
"duration_ms": 48.62,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 35.14,
|
||||
"duration_ms": 43.77,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 35.63,
|
||||
"duration_ms": 43.21,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 34.11,
|
||||
"duration_ms": 40.08,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 29.98,
|
||||
"duration_ms": 46.12,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 30.13,
|
||||
"duration_ms": 43.65,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 30.48,
|
||||
"duration_ms": 40.54,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 30.28,
|
||||
"duration_ms": 40.25,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 35.96 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 34.37 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 35.25 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 35.14 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 35.63 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 34.11 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 29.98 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 30.13 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 30.48 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 30.28 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 46.38 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 45.36 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 48.62 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.77 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 43.21 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 40.08 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 46.12 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 43.65 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 40.54 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 40.25 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"root": ".",
|
||||
"summary": {
|
||||
"target_python": "3.11",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Python Compatibility
|
||||
|
||||
Generated at: `2026-06-16`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -58,11 +58,11 @@
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-16"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"index": {
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"package_count": 1,
|
||||
"packages": [
|
||||
{
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336`
|
||||
- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f`
|
||||
- Archive SHA256: `996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81`
|
||||
- Install simulated: `True`
|
||||
|
||||
|
||||
+16
-16
File diff suppressed because one or more lines are too long
+231
-215
File diff suppressed because it is too large
Load Diff
+35
-32
@@ -412,7 +412,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_pass_count": 7,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -471,8 +471,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -997,12 +996,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1",
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1018,14 +1017,14 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 64
|
||||
},
|
||||
"commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1130,7 +1129,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 213,
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1168,7 +1167,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1205,7 +1204,7 @@
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-16"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": []
|
||||
@@ -1300,7 +1299,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1317,14 +1316,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1333,7 +1332,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1440,7 +1439,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1558,7 +1559,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1610,8 +1611,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -1630,7 +1631,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1950,7 +1953,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1987,7 +1990,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2008,8 +2011,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2025,8 +2028,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"source": "reports/review_annotations_input.json",
|
||||
"source": "tests/tmp_review_studio/empty_review_annotations_input.json",
|
||||
"summary": {
|
||||
"annotation_count": 0,
|
||||
"open_count": 0,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"summary": {
|
||||
"waiver_count": 0,
|
||||
"active_count": 0,
|
||||
@@ -52,7 +52,7 @@
|
||||
"Reviewer links output_review_adjudication or output_execution evidence."
|
||||
],
|
||||
"suggested_evidence": "reports/output_review_adjudication.md",
|
||||
"suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"<reviewer>\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md",
|
||||
"suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"<reviewer>\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md",
|
||||
"world_class_boundary": "Does not count as provider, human, or public world-class completion evidence."
|
||||
},
|
||||
{
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
- waiver allowed: `true`
|
||||
- risk: review pending 5; model-executed 0; output failures 0
|
||||
- evidence: `reports/output_review_adjudication.md`
|
||||
- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md`
|
||||
- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md`
|
||||
- world-class boundary: Does not count as provider, human, or public world-class completion evidence.
|
||||
|
||||
#### Required Review
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"package_dir": "dist",
|
||||
"package_dir": "tests/tmp_review_studio/dist",
|
||||
"expected_capabilities": [
|
||||
"file_write",
|
||||
"network",
|
||||
@@ -42,7 +42,7 @@
|
||||
{
|
||||
"target": "openai",
|
||||
"status": "pass",
|
||||
"adapter": "dist/targets/openai/adapter.json",
|
||||
"adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json",
|
||||
"permission_model": "metadata-only",
|
||||
"native_enforcement": false,
|
||||
"metadata_fallback_explicit": true,
|
||||
@@ -139,7 +139,7 @@
|
||||
{
|
||||
"target": "claude",
|
||||
"status": "pass",
|
||||
"adapter": "dist/targets/claude/adapter.json",
|
||||
"adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json",
|
||||
"permission_model": "neutral-source-plus-adapter",
|
||||
"native_enforcement": false,
|
||||
"metadata_fallback_explicit": true,
|
||||
@@ -221,7 +221,7 @@
|
||||
{
|
||||
"target": "generic",
|
||||
"status": "pass",
|
||||
"adapter": "dist/targets/generic/adapter.json",
|
||||
"adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json",
|
||||
"permission_model": "agent-skills-compatible-metadata",
|
||||
"native_enforcement": false,
|
||||
"metadata_fallback_explicit": true,
|
||||
@@ -303,7 +303,7 @@
|
||||
{
|
||||
"target": "vscode",
|
||||
"status": "pass",
|
||||
"adapter": "dist/targets/vscode/adapter.json",
|
||||
"adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json",
|
||||
"permission_model": "vscode-workspace-trust-plus-metadata",
|
||||
"native_enforcement": false,
|
||||
"metadata_fallback_explicit": true,
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 213,
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `213`
|
||||
- Package SHA256: `db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336`
|
||||
- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f`
|
||||
|
||||
## Failures
|
||||
|
||||
|
||||
@@ -919,7 +919,7 @@
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>6 / 13</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li><li><span data-lang="zh-CN">采用样本</span><span data-lang="en">Adoption sample</span></li></ul></article></div></article>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>7 / 13</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li></ul></article></div></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -412,7 +412,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_pass_count": 7,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -471,8 +471,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1001,12 +1000,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1",
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1022,14 +1021,14 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 64
|
||||
},
|
||||
"commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1134,7 +1133,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 213,
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1172,7 +1171,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1209,7 +1208,7 @@
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-16"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": []
|
||||
@@ -1304,7 +1303,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1321,14 +1320,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1337,7 +1336,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1444,7 +1443,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1562,7 +1563,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1614,8 +1615,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -1634,7 +1635,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1954,7 +1957,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1991,7 +1994,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2012,8 +2015,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2029,8 +2032,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -919,7 +919,7 @@
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>6 / 13</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li><li><span data-lang="zh-CN">采用样本</span><span data-lang="en">Adoption sample</span></li></ul></article></div></article>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>7 / 13</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li></ul></article></div></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
+35
-32
@@ -411,7 +411,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_pass_count": 7,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -470,8 +470,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -996,12 +995,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "7ef645993fc40e21528099046d09056d87af293c171850a84acbcab4d7f8f0a1",
|
||||
"source_contract_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"evidence_bundle_sha256": "95c6335a413934f197ca1595747c312701edb6028e8fc2ffed6cc4755c0a8d1f",
|
||||
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1017,14 +1016,14 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 13,
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"world_class_source_pass_count": 7,
|
||||
"world_class_source_blocked_count": 6,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 64
|
||||
},
|
||||
"commit": "4b2767900bbbbb962d2b11a92610951c3a0a0f0e",
|
||||
"commit": "54851019f53f3ee3f508dedfc0daaf192d6bc6ad",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
@@ -1129,7 +1128,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 213,
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1167,7 +1166,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336",
|
||||
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
|
||||
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1204,7 +1203,7 @@
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-16"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": []
|
||||
@@ -1299,7 +1298,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1316,14 +1315,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1332,7 +1331,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1439,7 +1438,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1557,7 +1558,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1609,8 +1610,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -1629,7 +1630,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -1949,7 +1952,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1986,7 +1989,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2007,8 +2010,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2024,8 +2027,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
},
|
||||
"catalog": {
|
||||
"workspace_root": ".",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"skills": [
|
||||
{
|
||||
"name": "yao-meta-skill",
|
||||
@@ -255,8 +255,8 @@
|
||||
"report": "reports/adoption_drift_report.json",
|
||||
"risk_band": "low",
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"candidate_count": 0
|
||||
}
|
||||
},
|
||||
@@ -919,7 +919,7 @@
|
||||
"name": "incident-command-governor",
|
||||
"path": "examples/governed-incident-command/generated-skill",
|
||||
"reason": "review overdue by cadence monthly",
|
||||
"age_days": 77,
|
||||
"age_days": 74,
|
||||
"allowed_days": 31,
|
||||
"actionable": false,
|
||||
"scope": "example"
|
||||
|
||||
@@ -351,7 +351,7 @@
|
||||
"key": "native-client-telemetry",
|
||||
"label": "Native Client Telemetry",
|
||||
"status": "external_required",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"target": "A real Browser/Chrome/provider client sends production metadata events",
|
||||
"evidence": [
|
||||
{
|
||||
@@ -433,7 +433,7 @@
|
||||
"key": "native-client-telemetry",
|
||||
"label": "Native Client Telemetry",
|
||||
"status": "external_required",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"target": "A real Browser/Chrome/provider client sends production metadata events",
|
||||
"evidence": [
|
||||
{
|
||||
|
||||
@@ -30,7 +30,7 @@ Generated at: `2026-06-16`
|
||||
| Registry Distribution | pass | zip entries 647; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
|
||||
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
|
||||
| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
|
||||
| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
## Open Highest-Leverage Gaps
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"local_blueprint_ready": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"writes_source_files": false,
|
||||
"auto_patch_enabled": false,
|
||||
@@ -31,7 +31,7 @@
|
||||
"local_blueprint_ready": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"writes_source_files": false,
|
||||
"auto_patch_enabled": false,
|
||||
@@ -480,7 +480,7 @@
|
||||
"local_blueprint_ready": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true
|
||||
},
|
||||
"actions": [
|
||||
|
||||
@@ -14,7 +14,7 @@ Generated at: `2026-06-16`
|
||||
- local blueprint ready: `true`
|
||||
- public world-class ready: `false`
|
||||
- world-class pending: `4`
|
||||
- release lock ready: `true`
|
||||
- release lock ready: `false`
|
||||
- evidence consistency ok: `true`
|
||||
|
||||
This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence.
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"top_score": 88,
|
||||
"skill_count": 12,
|
||||
"actionable_portfolio_issue_count": 0,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
@@ -35,7 +35,7 @@
|
||||
"top_score": 88,
|
||||
"skill_count": 12,
|
||||
"actionable_portfolio_issue_count": 0,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
@@ -287,7 +287,7 @@
|
||||
"no_route_opportunity_count": 0
|
||||
},
|
||||
"release_state": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4
|
||||
|
||||
@@ -13,7 +13,7 @@ Week: `2026-W25`
|
||||
- top_score: `88`
|
||||
- skill_count: `12`
|
||||
- actionable_portfolio_issue_count: `0`
|
||||
- release_lock_ready: `True`
|
||||
- release_lock_ready: `False`
|
||||
- evidence_consistency_ok: `True`
|
||||
- public_world_class_ready: `False`
|
||||
- world_class_pending_count: `4`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"previous": {
|
||||
"name": "yao-meta-skill",
|
||||
"version": "1.0.0",
|
||||
@@ -75,7 +75,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "db36e8f9d7d6745de28566d93383ac8d491806ee1e7a7175e6611db8e2958336"
|
||||
"to": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -90,7 +90,7 @@
|
||||
"artifacts": {
|
||||
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
|
||||
"current_package": "reports/registry_audit.json",
|
||||
"json": "reports/upgrade_check.json",
|
||||
"markdown": "reports/upgrade_check.md"
|
||||
"json": "tests/tmp_review_studio/upgrade_check.json",
|
||||
"markdown": "tests/tmp_review_studio/upgrade_check.md"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,7 +113,9 @@
|
||||
"observed token metadata"
|
||||
],
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -310,7 +312,7 @@
|
||||
"source_accepted": false,
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
|
||||
@@ -83,7 +83,9 @@ This report validates the intake contract for human and external evidence. A val
|
||||
|
||||
#### Source Runbook
|
||||
|
||||
- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
|
||||
- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}`
|
||||
- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- `python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>`
|
||||
- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
|
||||
- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -31,8 +31,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -76,7 +76,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -396,7 +398,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -433,7 +435,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -454,8 +456,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -471,8 +473,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -8,8 +8,8 @@ Generated at: `2026-06-16`
|
||||
- ready to claim world-class: `false`
|
||||
- entries: `4`
|
||||
- source accepted: `0`
|
||||
- source checks: `6` pass / `13` total
|
||||
- source blocked: `7`
|
||||
- source checks: `7` pass / `13` total
|
||||
- source blocked: `6`
|
||||
- accepted: `0`
|
||||
- pending: `4`
|
||||
- human pending: `1`
|
||||
@@ -29,7 +29,7 @@ This ledger records the current evidence state. It requires both passing source
|
||||
| `provider-holdout` | `pending` | `missing` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
|
||||
| `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices in the decision template, then regenerate adjudication. |
|
||||
| `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
|
||||
| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. |
|
||||
| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
@@ -47,7 +47,9 @@ This ledger records the current evidence state. It requires both passing source
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
|
||||
- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}`
|
||||
- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- `python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>`
|
||||
- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
|
||||
- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
|
||||
@@ -169,8 +171,8 @@ This ledger records the current evidence state. It requires both passing source
|
||||
|
||||
- objective: Import production metadata-only events from a real external client into the local drift loop.
|
||||
- source status: `external_required`
|
||||
- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
|
||||
- source checks: `1` pass / `3` total
|
||||
- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
|
||||
- source checks: `2` pass / `3` total
|
||||
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}`
|
||||
|
||||
### Provenance Requirements
|
||||
@@ -194,7 +196,7 @@ This ledger records the current evidence state. It requires both passing source
|
||||
| Check | Current | Expected | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` |
|
||||
| Adoption sample | `0` | `>0` | `blocked` |
|
||||
| Adoption sample | `1` | `>0` | `pass` |
|
||||
| Raw content blocked | `False` | `false` | `pass` |
|
||||
|
||||
### Completion Assertions
|
||||
|
||||
@@ -26,7 +26,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -144,7 +146,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -187,7 +189,9 @@
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -305,7 +309,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
|
||||
@@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It
|
||||
| `provider-holdout` | `external_required` | `external` | operator with provider credentials | model-executed 0; token-observed 0 |
|
||||
| `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 |
|
||||
| `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 |
|
||||
| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 |
|
||||
| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
@@ -31,7 +31,9 @@ This report is an execution plan for the remaining world-class evidence gaps. It
|
||||
|
||||
### Runbook
|
||||
|
||||
- `YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
|
||||
- `export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}`
|
||||
- `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60`
|
||||
- `python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>`
|
||||
- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
|
||||
- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
|
||||
|
||||
@@ -65,7 +65,7 @@
|
||||
<span class="eyebrow">Evidence Collection</span>
|
||||
<h1>World-Class Evidence Preflight</h1>
|
||||
<p class="lede">This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.</p>
|
||||
<div class="stats"><article><span>Decision</span><strong>collection-preflight-blocked</strong></article><article><span>Pending</span><strong>4</strong></article><article><span>Ready</span><strong>1</strong></article><article><span>Blocked</span><strong>3</strong></article><article><span>Source</span><strong>6/13</strong></article></div>
|
||||
<div class="stats"><article><span>Decision</span><strong>collection-preflight-blocked</strong></article><article><span>Pending</span><strong>4</strong></article><article><span>Ready</span><strong>1</strong></article><article><span>Blocked</span><strong>3</strong></article><article><span>Source</span><strong>7/13</strong></article></div>
|
||||
</section>
|
||||
<section class="section two-col" id="handoff">
|
||||
<article class="panel">
|
||||
@@ -234,7 +234,7 @@
|
||||
</section>
|
||||
<section class="runbook">
|
||||
<h4>Runbook</h4>
|
||||
<ul><li>YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul>
|
||||
<ul><li>Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.</li><li>export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}</li><li>python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul>
|
||||
</section>
|
||||
</article>
|
||||
|
||||
@@ -611,15 +611,15 @@
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="check-row blocked">
|
||||
<article class="check-row pass">
|
||||
<div>
|
||||
<span>adoption_sample_count</span>
|
||||
<strong>Adoption sample</strong>
|
||||
</div>
|
||||
<dl>
|
||||
<dt>Current</dt><dd><code>0</code></dd>
|
||||
<dt>Current</dt><dd><code>1</code></dd>
|
||||
<dt>Expected</dt><dd><code>>0</code></dd>
|
||||
<dt>Status</dt><dd>blocked</dd>
|
||||
<dt>Status</dt><dd>pass</dd>
|
||||
<dt>Action</dt><dd>Telemetry must include adoption outcome evidence.</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
"collection_ready_count": 1,
|
||||
"collection_blocked_count": 3,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"pending_count": 4,
|
||||
"ready_to_claim_world_class": false,
|
||||
"credential_value_exposed": false,
|
||||
@@ -173,7 +173,9 @@
|
||||
"artifact_prefill_counts_as_evidence": false
|
||||
},
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -533,8 +535,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -889,8 +891,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ Generated at: `2026-06-16`
|
||||
- credential value exposed: `false`
|
||||
- collection ready: `1`
|
||||
- collection blocked: `3`
|
||||
- source checks: `6` pass / `13` total
|
||||
- source checks: `7` pass / `13` total
|
||||
|
||||
This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.
|
||||
|
||||
@@ -150,7 +150,7 @@ Generate the submission kit after the real provider, human, native-permission, o
|
||||
| Check | Current | Expected | Status | Next action |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. |
|
||||
| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. |
|
||||
| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. |
|
||||
| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. |
|
||||
|
||||
## Boundary
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
<span class="eyebrow">Evidence Operations</span>
|
||||
<h1>World-Class Operator Runbook</h1>
|
||||
<p class="lede">A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.</p>
|
||||
<div class="stats"><article><span>Pending</span><strong>4</strong></article><article><span>Awaiting</span><strong>4</strong></article><article><span>Ready</span><strong>0</strong></article><article><span>Source</span><strong>6/13</strong></article><article><span>Blocked</span><strong>7</strong></article><article><span>Invalid</span><strong>0</strong></article></div>
|
||||
<div class="stats"><article><span>Pending</span><strong>4</strong></article><article><span>Awaiting</span><strong>4</strong></article><article><span>Ready</span><strong>0</strong></article><article><span>Source</span><strong>7/13</strong></article><article><span>Blocked</span><strong>6</strong></article><article><span>Invalid</span><strong>0</strong></article></div>
|
||||
</section>
|
||||
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
|
||||
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid"><article class="item-card awaiting-submission">
|
||||
@@ -69,7 +69,7 @@
|
||||
<dt>Blocked</dt><dd><code>2</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/provider-holdout.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.</li><li>export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}</li><li>python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
|
||||
<div class="mini-grid">
|
||||
<section><h4>Must Collect</h4><ul><li>provider-backed model run</li><li>observed timing</li><li>observed token metadata</li></ul></section>
|
||||
@@ -120,7 +120,7 @@
|
||||
<dl>
|
||||
<dt>Owner</dt><dd>Browser/Chrome/IDE/provider client integrator</dd>
|
||||
<dt>Ledger</dt><dd><code>pending</code></dd>
|
||||
<dt>Blocked</dt><dd><code>2</code></dd>
|
||||
<dt>Blocked</dt><dd><code>1</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/native-client-telemetry.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/</li><li>Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.</li><li>python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl</li><li>python3 scripts/yao.py skill-atlas --workspace-root .</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
@@ -130,8 +130,8 @@
|
||||
<section><h4>Success Checks</h4><ul><li>reports/adoption_drift_report.json summary.source_types.external > 0</li><li>reports/adoption_drift_report.json summary.adoption_sample_count > 0</li><li>reports/skill_os2_audit.json item native-client-telemetry status becomes pass</li></ul></section>
|
||||
<section><h4>Privacy</h4><ul><li>Telemetry must remain metadata-only and local-first.</li><li>Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.</li></ul></section>
|
||||
</div>
|
||||
<section class="source-panel"><h4>Next Source Actions</h4><ul><li>Import at least one metadata-only event from a real client.</li><li>Telemetry must include adoption outcome evidence.</li></ul></section>
|
||||
<section class="source-panel"><h4>Source Evidence Snapshot</h4><ul class='source-checks'><li class='source-check blocked'><span>External events</span><code>external_source_events: 0 / >0</code><small>Import at least one metadata-only event from a real client.</small></li><li class='source-check blocked'><span>Adoption sample</span><code>adoption_sample_count: 0 / >0</code><small>Telemetry must include adoption outcome evidence.</small></li><li class='source-check pass'><span>Raw content blocked</span><code>raw_content_allowed: False / false</code><small>Telemetry must stay metadata-only.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Next Source Actions</h4><ul><li>Import at least one metadata-only event from a real client.</li></ul></section>
|
||||
<section class="source-panel"><h4>Source Evidence Snapshot</h4><ul class='source-checks'><li class='source-check blocked'><span>External events</span><code>external_source_events: 0 / >0</code><small>Import at least one metadata-only event from a real client.</small></li><li class='source-check pass'><span>Adoption sample</span><code>adoption_sample_count: 1 / >0</code><small>Telemetry must include adoption outcome evidence.</small></li><li class='source-check pass'><span>Raw content blocked</span><code>raw_content_allowed: False / false</code><small>Telemetry must stay metadata-only.</small></li></ul></section>
|
||||
</article></div></section>
|
||||
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for submission review; ledger review still requires passing source evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
|
||||
</main>
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
"valid_packet_source_incomplete_count": 0,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"runbook_counts_as_completion": false,
|
||||
"decision": "collect-evidence"
|
||||
@@ -35,7 +35,9 @@
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"execution_runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
|
||||
@@ -397,7 +399,7 @@
|
||||
"review_state": "awaiting-submission",
|
||||
"source_accepted": false,
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"execution_runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
|
||||
@@ -445,7 +447,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -466,8 +468,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -482,10 +484,9 @@
|
||||
"next_action": "Telemetry must stay metadata-only."
|
||||
}
|
||||
],
|
||||
"blocked_source_check_count": 2,
|
||||
"blocked_source_check_count": 1,
|
||||
"next_source_actions": [
|
||||
"Import at least one metadata-only event from a real client.",
|
||||
"Telemetry must include adoption outcome evidence."
|
||||
"Import at least one metadata-only event from a real client."
|
||||
],
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
|
||||
@@ -28,7 +28,7 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials. | operator with provider credentials |
|
||||
| `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Record a reviewer choice for every pair. | human reviewer |
|
||||
| `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator |
|
||||
| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator |
|
||||
| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
@@ -40,7 +40,9 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
|
||||
- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
|
||||
- export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
|
||||
- python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
|
||||
- python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
|
||||
- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
|
||||
- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
|
||||
@@ -242,7 +244,7 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
|
||||
- objective: Import production metadata-only events from a real external client into the local drift loop.
|
||||
- blocking reason: No evidence packet has been submitted for review.
|
||||
- blocked source checks: `2`
|
||||
- blocked source checks: `1`
|
||||
- submission: `evidence/world_class/submissions/native-client-telemetry.json`
|
||||
- template: `evidence/world_class/templates/native-client-telemetry.intake.json`
|
||||
|
||||
@@ -295,14 +297,13 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
### Next Source Actions
|
||||
|
||||
- Import at least one metadata-only event from a real client.
|
||||
- Telemetry must include adoption outcome evidence.
|
||||
|
||||
### Source Evidence Snapshot
|
||||
|
||||
| Check | Current | Expected | Status | Next action |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. |
|
||||
| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. |
|
||||
| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. |
|
||||
| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. |
|
||||
|
||||
## Boundary
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
"unmatched_submission_count": 0,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"review_counts_submission_as_completion": false,
|
||||
"summary": {
|
||||
"review_item_count": 4,
|
||||
@@ -27,8 +27,8 @@
|
||||
"unmatched_submission_count": 0,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 13,
|
||||
"source_pass_count": 6,
|
||||
"source_blocked_count": 7,
|
||||
"source_pass_count": 7,
|
||||
"source_blocked_count": 6,
|
||||
"ready_to_claim_world_class": false,
|
||||
"review_counts_submission_as_completion": false,
|
||||
"decision": "awaiting-submissions"
|
||||
@@ -302,7 +302,7 @@
|
||||
"intake_errors": [],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -323,8 +323,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -340,8 +340,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"success_checks": [
|
||||
"reports/adoption_drift_report.json summary.source_types.external > 0",
|
||||
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
|
||||
|
||||
@@ -139,7 +139,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w
|
||||
#### Source Checks
|
||||
|
||||
- External events: 0 / >0 => blocked
|
||||
- Adoption sample: 0 / >0 => blocked
|
||||
- Adoption sample: 1 / >0 => pass
|
||||
- Raw content blocked: False / false => pass
|
||||
|
||||
#### Completion Assertions
|
||||
|
||||
@@ -17,7 +17,9 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = {
|
||||
"owner": "operator with provider credentials",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
|
||||
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
|
||||
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
],
|
||||
"success_checks": [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"workspace_root": ".",
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-13",
|
||||
"skills": [
|
||||
{
|
||||
"name": "yao-meta-skill",
|
||||
@@ -211,8 +211,8 @@
|
||||
"report": "reports/adoption_drift_report.json",
|
||||
"risk_band": "low",
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"adoption_rate": 100.0,
|
||||
"candidate_count": 0
|
||||
}
|
||||
},
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
"name": "incident-command-governor",
|
||||
"path": "examples/governed-incident-command/generated-skill",
|
||||
"reason": "review overdue by cadence monthly",
|
||||
"age_days": 77,
|
||||
"age_days": 74,
|
||||
"allowed_days": 31,
|
||||
"actionable": false,
|
||||
"scope": "example"
|
||||
|
||||
@@ -545,6 +545,7 @@ def main() -> None:
|
||||
assert any("world-class-intake" in item["command"] for item in provider_action_step["commands"]), provider_action_step
|
||||
assert any("world-class-ledger" in item["command"] for item in provider_action_step["commands"]), provider_action_step
|
||||
assert any("output-exec --provider-runner openai" in item for item in provider_action_step["runbook"]), provider_action_step
|
||||
assert not any("<redacted>" in item or "OPENAI_API_KEY=" in item for item in provider_action_step["runbook"]), provider_action_step
|
||||
assert "provider-backed model run" in provider_action_step["provenance_requirements"], provider_action_step
|
||||
assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider_action_step["success_checks"], provider_action_step
|
||||
assert "reports/output_execution_runs.json" in provider_action_step["evidence_artifacts"], provider_action_step
|
||||
@@ -727,6 +728,7 @@ def main() -> None:
|
||||
assert "计划、metadata fallback、待评审和本地命令不会被当成完成证据" in html, html
|
||||
assert "执行步骤" in html, html
|
||||
assert "output-exec --provider-runner openai" in html, html
|
||||
assert "<redacted>" not in html and "<redacted>" not in html, html
|
||||
assert "world-runbook-panel" in html, html
|
||||
assert "源证据检查" in html, html
|
||||
assert "world-source-checks" in html, html
|
||||
|
||||
@@ -604,6 +604,8 @@ def main() -> None:
|
||||
assert "`python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`" in markdown, markdown
|
||||
assert "Source Runbook" in markdown, markdown
|
||||
assert "output-exec --provider-runner openai" in markdown, markdown
|
||||
assert "<redacted>" not in markdown, markdown
|
||||
assert "OPENAI_API_KEY=<redacted>" not in markdown, markdown
|
||||
assert "Templates and planned work do not count as accepted evidence." in markdown, markdown
|
||||
assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown
|
||||
assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown
|
||||
@@ -669,6 +671,9 @@ def main() -> None:
|
||||
assert kit_draft["template_only"] is True, kit_draft
|
||||
assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft
|
||||
assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft
|
||||
assert kit_draft["provenance"]["run_command"] == "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", kit_draft
|
||||
assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft
|
||||
assert "<redacted>" not in json.dumps(kit_draft), kit_draft
|
||||
kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8"))
|
||||
assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"]
|
||||
assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"]
|
||||
@@ -680,6 +685,7 @@ def main() -> None:
|
||||
assert "Drafts are not accepted evidence." in kit_readme, kit_readme
|
||||
assert "Execution Runbook" in kit_readme, kit_readme
|
||||
assert "output-exec --provider-runner openai" in kit_readme, kit_readme
|
||||
assert "<redacted>" not in kit_readme, kit_readme
|
||||
assert "validate intake" in kit_readme, kit_readme
|
||||
assert "Artifact Checklist" in kit_readme, kit_readme
|
||||
assert "Evidence Matrix" in kit_readme, kit_readme
|
||||
@@ -715,6 +721,7 @@ def main() -> None:
|
||||
assert "World-Class Evidence Submission Kit" in kit_html, kit_html
|
||||
assert "Execution Runbook" in kit_html, kit_html
|
||||
assert "output-exec --provider-runner openai" in kit_html, kit_html
|
||||
assert "<redacted>" not in kit_html and "<redacted>" not in kit_html, kit_html
|
||||
assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html
|
||||
assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ def main() -> None:
|
||||
}, entries
|
||||
assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"]
|
||||
assert any("output-exec --provider-runner openai" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"]
|
||||
assert not any("<redacted>" in step or "OPENAI_API_KEY=" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"]
|
||||
provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]}
|
||||
assert provider_source["model_executed_count"]["status"] == "blocked", provider_source
|
||||
assert provider_source["timing_observed_count"]["status"] == "pass", provider_source
|
||||
@@ -147,6 +148,8 @@ def main() -> None:
|
||||
assert "source checks:" in markdown, markdown
|
||||
assert "Source Runbook" in markdown, markdown
|
||||
assert "output-exec --provider-runner openai" in markdown, markdown
|
||||
assert "<redacted>" not in markdown, markdown
|
||||
assert "OPENAI_API_KEY=<redacted>" not in markdown, markdown
|
||||
assert "Source Evidence Checks" in markdown, markdown
|
||||
assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown
|
||||
assert "`provider-holdout`" in markdown, markdown
|
||||
|
||||
@@ -54,7 +54,11 @@ def main() -> None:
|
||||
"native-client-telemetry",
|
||||
}, tasks
|
||||
assert set(requirements) == set(tasks), requirements
|
||||
assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
provider_runbook = tasks["provider-holdout"]["runbook"]
|
||||
assert any("--provider-runner openai" in command for command in provider_runbook), tasks["provider-holdout"]
|
||||
assert any(command.startswith("Set OPENAI_API_KEY") for command in provider_runbook), tasks["provider-holdout"]
|
||||
assert any(command.startswith("export YAO_OUTPUT_EVAL_MODEL=") for command in provider_runbook), tasks["provider-holdout"]
|
||||
assert not any("<redacted>" in command or "OPENAI_API_KEY=" in command for command in provider_runbook), tasks["provider-holdout"]
|
||||
assert any("world-class-intake . --submissions-dir evidence/world_class/submissions" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
assert "reports/output_review_decisions.json" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"]
|
||||
@@ -76,6 +80,8 @@ def main() -> None:
|
||||
assert "`provider-holdout`" in markdown, markdown
|
||||
assert "ready to claim world-class: `false`" in markdown, markdown
|
||||
assert "ledger completion required: `true`" in markdown, markdown
|
||||
assert "<redacted>" not in markdown, markdown
|
||||
assert "OPENAI_API_KEY=<redacted>" not in markdown, markdown
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ def main() -> None:
|
||||
assert provider["source_accepted"] is False, provider
|
||||
assert provider["blocked_source_check_count"] == 2, provider
|
||||
assert any("output-exec --provider-runner openai" in step for step in provider["execution_runbook"]), provider
|
||||
assert not any("<redacted>" in step or "OPENAI_API_KEY=" in step for step in provider["execution_runbook"]), provider
|
||||
assert "Run provider-backed output-exec with real credentials." in provider["next_source_actions"], provider
|
||||
assert "Provider execution should return non-estimated token usage." in provider["next_source_actions"], provider
|
||||
assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider
|
||||
@@ -143,6 +144,8 @@ def main() -> None:
|
||||
assert "| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials." in markdown, markdown
|
||||
assert "Source Runbook" in markdown, markdown
|
||||
assert "output-exec --provider-runner openai" in markdown, markdown
|
||||
assert "<redacted>" not in markdown, markdown
|
||||
assert "OPENAI_API_KEY=<redacted>" not in markdown, markdown
|
||||
assert "### Next Source Actions" in markdown, markdown
|
||||
assert "- Provider execution should return non-estimated token usage." in markdown, markdown
|
||||
assert "Source Evidence Snapshot" in markdown, markdown
|
||||
@@ -159,6 +162,8 @@ def main() -> None:
|
||||
assert "Next Source Actions" in html, html
|
||||
assert "Source Runbook" in html, html
|
||||
assert "output-exec --provider-runner openai" in html, html
|
||||
assert "<redacted>" not in html and "<redacted>" not in html, html
|
||||
assert "OPENAI_API_KEY=<redacted>" not in html, html
|
||||
assert "Source Evidence Snapshot" in html, html
|
||||
assert "model_executed_count" in html, html
|
||||
assert "model_executed_count: 0 / >0" in html, html
|
||||
|
||||
Reference in New Issue
Block a user