frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add world-class evidence intake checklist

Browse Source
This commit is contained in:
yaojingang
2026-06-14 05:29:26 +08:00
parent 088904f519
commit 810885d70e
31 changed files with 1119 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evidence/world_class/README.md
+2
View File
@@ -18,4 +18,6 @@ The intake validator checks:
- credentials, secrets, raw user content, and raw provider prompts are explicitly excluded
- planned work, local command-only output, and metadata fallback are not claimed as completion evidence
The generated intake report also includes an `operator_checklist` for each pending evidence item. Use it to find the template path, target submission path, preparation command, validation command, required provenance, success checks, and privacy boundary before asking a reviewer or external operator to submit evidence.
Accepted intake means "ready for ledger review", not "world-class complete". The ledger remains the source of truth for `ready_to_claim_world_class`.
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/adoption_drift_report.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13T21:15:27Z",
"generated_at": "2026-06-13T21:28:44Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
reports/architecture_maintainability.json
+9 -9
View File
@@ -12,7 +12,7 @@
"command_handler_count": 33,
"warn_line_threshold": 900,
"block_line_threshold": 1500,
"largest_file_lines": 887,
"largest_file_lines": 895,
"hotspot_count": 0,
"blocker_count": 0,
"decision": "pass"
@@ -20,11 +20,18 @@
"largest_files": [
{
"path": "scripts/render_review_studio.py",
"lines": 887,
"lines": 895,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Move data loading and large section renderers into focused review_studio_* modules."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 845,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/yao.py",
"lines": 839,
@@ -32,13 +39,6 @@
"severity": "pass",
"recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 838,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/skill_report_model.py",
"lines": 782,
reports/architecture_maintainability.md
+3 -3
View File
@@ -11,7 +11,7 @@ Generated at: `2026-06-14`
- internal modules: `18`
- CLI scripts: `72`
- Yao CLI command handlers: `33`
- largest file lines: `887`
- largest file lines: `895`
- hotspots: `0`
- blockers: `0`
@@ -25,9 +25,9 @@ No file-size hotspots found.
| File | Lines | Kind | Severity |
| --- | ---: | --- | --- |
| `scripts/render_review_studio.py` | `887` | `cli-script` | `pass` |
| `scripts/render_review_studio.py` | `895` | `cli-script` | `pass` |
| `tests/verify_yao_cli.py` | `845` | `test` | `pass` |
| `scripts/yao.py` | `839` | `cli-script` | `pass` |
| `tests/verify_yao_cli.py` | `838` | `test` | `pass` |
| `scripts/skill_report_model.py` | `782` | `internal-module` | `pass` |
| `scripts/compile_skill.py` | `734` | `cli-script` | `pass` |
| `scripts/optimize_description.py` | `723` | `cli-script` | `pass` |
reports/benchmark_reproducibility.json
+22 -22
View File
@@ -3,24 +3,24 @@
"ok": true,
"generated_at": "2026-06-14",
"skill_dir": ".",
"commit": "628acc5c35a9cf4cd0d3d684be50ee27724150ba",
"commit": "088904f519e04981f30c5c8c225de90a4aee2682",
"git_status": {
"available": true,
"dirty": true,
"changed_file_count": 36,
"changed_file_count": 31,
"sample": [
"MM registry/index.json",
"MM registry/packages/yao-meta-skill.json",
"MM reports/adoption_drift_report.json",
"M reports/architecture_maintainability.json",
"M reports/architecture_maintainability.md",
"MM reports/benchmark_reproducibility.json",
"MM reports/benchmark_reproducibility.md",
"MM reports/context_budget.json",
"M reports/install_simulation.json",
"MM reports/output_execution_runs.json",
"MM reports/output_execution_runs.md",
"MM reports/package_verification.json"
" M evidence/world_class/README.md",
" M registry/index.json",
" M registry/packages/yao-meta-skill.json",
" M reports/adoption_drift_report.json",
" M reports/architecture_maintainability.json",
" M reports/architecture_maintainability.md",
" M reports/benchmark_reproducibility.json",
" M reports/benchmark_reproducibility.md",
" M reports/context_budget.json",
" M reports/output_execution_runs.json",
" M reports/output_execution_runs.md",
" M reports/package_verification.json"
]
},
"summary": {
@@ -42,7 +42,7 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 36
"changed_file_count": 31
},
"methodology": {
"path": "reports/benchmark_methodology.md",
@@ -115,8 +115,8 @@
"label": "output_execution",
"path": "reports/output_execution_runs.json",
"exists": true,
"bytes": 7964,
"sha256": "5d8dd9775201caadbb68d4514261bd17459cc580eea3d0b273c0034d4103f41a"
"bytes": 7966,
"sha256": "9bc795936ec9610a8a6f81e692c8839b9dcac917c9839e1b99c5460ba8919061"
},
{
"label": "blind_review",
@@ -151,7 +151,7 @@
"path": "reports/security_trust_report.json",
"exists": true,
"bytes": 91959,
"sha256": "e64df7a7d3a9f30dadc0133b96d2473dcdce458be6c3b7c1feaa3778bca3a864"
"sha256": "29451ca6127d5d049f04978928110e97935e100a0e12f125ef7b4d4aae129271"
},
{
"label": "python_compatibility",
@@ -165,14 +165,14 @@
"path": "reports/registry_audit.json",
"exists": true,
"bytes": 3183,
"sha256": "1aeda726a6302c6b37a36d179a34d17c93277d7c798b3fb15044ce1c6632494d"
"sha256": "6f2179a1a51a05a1e4455ed113fa2499e62d643a046aab2735fb516d6f2fad35"
},
{
"label": "package_verification",
"path": "reports/package_verification.json",
"exists": true,
"bytes": 19325,
"sha256": "9e12aa80888c3c2508ff0681bc1bddb5fcd308657f87695c02ced23dc214ff3f"
"sha256": "69864577d09542dc4b20c4976516c7a6f20bdb95d76dd38e8fcc4f88691bceba"
},
{
"label": "install_simulation",
@@ -206,8 +206,8 @@
"label": "world_class_evidence_intake",
"path": "reports/world_class_evidence_intake.json",
"exists": true,
"bytes": 1775,
"sha256": "124b9239d7f35022735090712aadf2bf1d4dfba28f6bcf18f956fa0a73fe4482"
"bytes": 12523,
"sha256": "6eae947a6e2f3721a8c616b58795554fe6cd778be62c6e6c34833563582a505d"
},
{
"label": "world_class_claim_guard",
reports/benchmark_reproducibility.md
+7 -7
View File
@@ -1,7 +1,7 @@
# Benchmark Reproducibility
Generated at: `2026-06-14`
Commit: `628acc5c35a9cf4cd0d3d684be50ee27724150ba`
Commit: `088904f519e04981f30c5c8c225de90a4aee2682`
Working tree dirty at generation: `true`
## Summary
@@ -16,7 +16,7 @@ Working tree dirty at generation: `true`
- provider evidence complete: `false`
- human review complete: `false`
- world-class ready: `false`
- changed files at generation: `36`
- changed files at generation: `31`
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
@@ -40,20 +40,20 @@ This report proves local benchmark reproducibility only. It keeps external provi
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
| output_execution | `reports/output_execution_runs.json` | present | `5d8dd9775201` |
| output_execution | `reports/output_execution_runs.json` | present | `9bc795936ec9` |
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
| review_adjudication | `reports/output_review_adjudication.json` | present | `ddd9af90d42e` |
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
| trust_report | `reports/security_trust_report.json` | present | `e64df7a7d3a9` |
| trust_report | `reports/security_trust_report.json` | present | `29451ca6127d` |
| python_compatibility | `reports/python_compatibility.json` | present | `f65de3392609` |
| registry_audit | `reports/registry_audit.json` | present | `1aeda726a630` |
| package_verification | `reports/package_verification.json` | present | `9e12aa80888c` |
| registry_audit | `reports/registry_audit.json` | present | `6f2179a1a51a` |
| package_verification | `reports/package_verification.json` | present | `69864577d095` |
| install_simulation | `reports/install_simulation.json` | present | `11985a19c3bc` |
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `0fcd7f82c65f` |
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `ed1274b7c18c` |
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `5ed043491c3d` |
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `124b9239d7f3` |
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `6eae947a6e2f` |
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `608c954d1af9` |
## Reproduction Commands
reports/context_budget.json
+2 -2
View File
@@ -6,9 +6,9 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 1084478,
"other_text_tokens": 1096197,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 1085229,
"estimated_total_text_tokens": 1096948,
"relevant_file_count": 477,
"unused_resource_dirs": [],
"quality_signal_points": 130,
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.86,
"duration_ms": 27.65,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.95,
"duration_ms": 27.96,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 32.1,
"duration_ms": 27.98,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.57,
"duration_ms": 28.3,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.44,
"duration_ms": 29.47,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.59,
"duration_ms": 28.72,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.84,
"duration_ms": 29.11,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.9,
"duration_ms": 28.16,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.78,
"duration_ms": 27.95,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.9,
"duration_ms": 28.03,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 31.86 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 31.95 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 32.1 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.57 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 31.44 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 30.59 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.84 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.9 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.78 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.9 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 27.65 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.96 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.98 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.3 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.47 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.72 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 29.11 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.16 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.95 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.03 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"archive_entry_count": 557,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653`
- Archive SHA256: `2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c`
- Archive SHA256: `a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653`
- Package SHA256: `f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20`
- Archive SHA256: `2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2`
- Install simulated: `True`
## Compatibility
reports/review-studio.html
+83 -6
View File
File diff suppressed because one or more lines are too long
reports/review-studio.json
+249 -40
View File
@@ -75,7 +75,7 @@
"key": "architecture-maintainability",
"label": "架构维护",
"status": "pass",
"detail": "145 Python files; 0 hotspots; 0 blockers; largest 887 lines; 33 CLI handlers",
"detail": "145 Python files; 0 hotspots; 0 blockers; largest 895 lines; 33 CLI handlers",
"evidence": "reports/architecture_maintainability.json",
"link": "architecture_maintainability.md"
},
@@ -1306,9 +1306,9 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 36
"changed_file_count": 31
},
"commit": "628acc5c35a9cf4cd0d3d684be50ee27724150ba",
"commit": "088904f519e04981f30c5c8c225de90a4aee2682",
"missing_artifacts": [],
"limitations": [
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
@@ -1404,7 +1404,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 173,
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
},
"skill_atlas": {
"skill_count": 12,
@@ -1442,8 +1442,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -1474,7 +1474,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1490,7 +1490,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"archive_entry_count": 557,
"failure_count": 0,
"warning_count": 0
@@ -1569,12 +1569,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"to": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"to": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
@@ -3691,7 +3691,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.86,
"duration_ms": 27.65,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3719,7 +3719,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.95,
"duration_ms": 27.96,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3742,7 +3742,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 32.1,
"duration_ms": 27.98,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3770,7 +3770,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.57,
"duration_ms": 28.3,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3793,7 +3793,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.44,
"duration_ms": 29.47,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3821,7 +3821,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.59,
"duration_ms": 28.72,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3844,7 +3844,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.84,
"duration_ms": 29.11,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3871,7 +3871,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.9,
"duration_ms": 28.16,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3894,7 +3894,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.78,
"duration_ms": 27.95,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3923,7 +3923,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.9,
"duration_ms": 28.03,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -9622,7 +9622,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 173,
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
},
"failures": [],
"warnings": [],
@@ -12832,7 +12832,7 @@
"command_handler_count": 33,
"warn_line_threshold": 900,
"block_line_threshold": 1500,
"largest_file_lines": 887,
"largest_file_lines": 895,
"hotspot_count": 0,
"blocker_count": 0,
"decision": "pass"
@@ -12840,11 +12840,18 @@
"largest_files": [
{
"path": "scripts/render_review_studio.py",
"lines": 887,
"lines": 895,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Move data loading and large section renderers into focused review_studio_* modules."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 845,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/yao.py",
"lines": 839,
@@ -12852,13 +12859,6 @@
"severity": "pass",
"recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 838,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/skill_report_model.py",
"lines": 782,
@@ -12938,9 +12938,9 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 1084478,
"other_text_tokens": 1096197,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 1085229,
"estimated_total_text_tokens": 1096948,
"relevant_file_count": 477,
"unused_resource_dirs": [],
"quality_signal_points": 130,
@@ -14789,7 +14789,7 @@
"adoption_drift": {
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13T21:15:27Z",
"generated_at": "2026-06-13T21:28:44Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
@@ -15168,6 +15168,8 @@
"submission_count": 0,
"valid_submission_count": 0,
"invalid_submission_count": 0,
"operator_checklist_count": 4,
"operator_checklist_ready_count": 0,
"ready_for_external_collection": true,
"ready_for_ledger_review": false,
"ready_to_claim_world_class": false,
@@ -15206,6 +15208,213 @@
}
],
"submissions": [],
"operator_checklist": [
{
"evidence_key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/provider-holdout.intake.json",
"submission_path": "evidence/world_class/submissions/provider-holdout.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/provider-holdout.intake.json evidence/world_class/submissions/provider-holdout.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/provider-holdout.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"evidence_key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/human-adjudication.intake.json",
"submission_path": "evidence/world_class/submissions/human-adjudication.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/human-adjudication.intake.json evidence/world_class/submissions/human-adjudication.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/human-adjudication.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"evidence_key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json",
"submission_path": "evidence/world_class/submissions/native-permission-enforcement.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-permission-enforcement.intake.json evidence/world_class/submissions/native-permission-enforcement.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real target or installer guard",
"native enforcement flag",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"security/permission_policy.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-permission-enforcement.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement."
},
{
"evidence_key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/native-client-telemetry.intake.json",
"submission_path": "evidence/world_class/submissions/native-client-telemetry.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-client-telemetry.intake.json evidence/world_class/submissions/native-client-telemetry.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-client-telemetry.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_ledger": {
"json": "reports/world_class_evidence_ledger.json",
"markdown": "reports/world_class_evidence_ledger.md",
@@ -15563,8 +15772,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -15595,7 +15804,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -15620,7 +15829,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
@@ -15643,7 +15852,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"archive_entry_count": 557,
"failure_count": 0,
"warning_count": 0
@@ -16651,12 +16860,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"to": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"to": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
reports/review-viewer.json
+9 -9
View File
@@ -906,9 +906,9 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 36
"changed_file_count": 31
},
"commit": "628acc5c35a9cf4cd0d3d684be50ee27724150ba",
"commit": "088904f519e04981f30c5c8c225de90a4aee2682",
"missing_artifacts": [],
"limitations": [
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
@@ -1004,7 +1004,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 173,
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
},
"skill_atlas": {
"skill_count": 12,
@@ -1042,8 +1042,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -1074,7 +1074,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1090,7 +1090,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"archive_entry_count": 557,
"failure_count": 0,
"warning_count": 0
@@ -1169,12 +1169,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"to": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"to": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 173,
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `173`
- Package SHA256: `ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c`
- Package SHA256: `f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20`
## Failures
reports/skill-overview.json
+9 -9
View File
@@ -905,9 +905,9 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 36
"changed_file_count": 31
},
"commit": "628acc5c35a9cf4cd0d3d684be50ee27724150ba",
"commit": "088904f519e04981f30c5c8c225de90a4aee2682",
"missing_artifacts": [],
"limitations": [
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
@@ -1003,7 +1003,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 173,
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
},
"skill_atlas": {
"skill_count": 12,
@@ -1041,8 +1041,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c",
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"package_sha256": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
"compatibility": {
"openai": "pass",
@@ -1073,7 +1073,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1089,7 +1089,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653",
"archive_sha256": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2",
"archive_entry_count": 557,
"failure_count": 0,
"warning_count": 0
@@ -1168,12 +1168,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"to": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"to": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "a35cc5a6ebbbb278f826506ff52274ec65c8c5ffbdaa9dee2ef477d9dc409653"
"to": "2c2f2d3a4fffe1dfc54e282c1058efc1d3476b7de588f8e9167b26a7428a63d2"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ac773f9145981816dd82890dd769bb608245fb14a5f6605db7e606d377ffcd9c"
"to": "f34bc1b155763d5269ae0ae904de35383fd4d0000006af8ad7a6c85ab7ce4b20"
}
]
},
reports/world_class_evidence_intake.json
+209
View File
@@ -11,6 +11,8 @@
"submission_count": 0,
"valid_submission_count": 0,
"invalid_submission_count": 0,
"operator_checklist_count": 4,
"operator_checklist_ready_count": 0,
"ready_for_external_collection": true,
"ready_for_ledger_review": false,
"ready_to_claim_world_class": false,
@@ -49,6 +51,213 @@
}
],
"submissions": [],
"operator_checklist": [
{
"evidence_key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/provider-holdout.intake.json",
"submission_path": "evidence/world_class/submissions/provider-holdout.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/provider-holdout.intake.json evidence/world_class/submissions/provider-holdout.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/provider-holdout.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"evidence_key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/human-adjudication.intake.json",
"submission_path": "evidence/world_class/submissions/human-adjudication.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/human-adjudication.intake.json evidence/world_class/submissions/human-adjudication.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/human-adjudication.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"evidence_key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json",
"submission_path": "evidence/world_class/submissions/native-permission-enforcement.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-permission-enforcement.intake.json evidence/world_class/submissions/native-permission-enforcement.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real target or installer guard",
"native enforcement flag",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"security/permission_policy.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-permission-enforcement.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement."
},
{
"evidence_key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"readiness": "awaiting-submission",
"blocking_reason": "No real evidence submission has been provided yet.",
"template_status": "pass",
"submission_status": "missing",
"template_path": "evidence/world_class/templates/native-client-telemetry.intake.json",
"submission_path": "evidence/world_class/submissions/native-client-telemetry.json",
"commands": {
"prepare_submission": "mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-client-telemetry.intake.json evidence/world_class/submissions/native-client-telemetry.json",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-client-telemetry.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
]
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_ledger": {
"json": "reports/world_class_evidence_ledger.json",
"markdown": "reports/world_class_evidence_ledger.md",
reports/world_class_evidence_intake.md
+163
View File
@@ -9,6 +9,7 @@ Generated at: `2026-06-14`
- templates: `4` / `4`
- submissions: `0` valid / `0` total
- invalid submissions: `0`
- operator checklist: `0` ready / `4` total
- ready for external collection: `true`
- ready for ledger review: `false`
- ready to claim world-class: `false`
@@ -31,6 +32,168 @@ This report validates the intake contract for human and external evidence. A val
| --- | --- | --- | --- |
| `none` | `n/a` | none | none |
## Operator Checklist
| Evidence | Readiness | Submission | Next action |
| --- | --- | --- | --- |
| `provider-holdout` | `awaiting-submission` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
| `human-adjudication` | `awaiting-submission` | `missing` | Record real A/B choices in the decision template, then regenerate adjudication. |
| `native-permission-enforcement` | `awaiting-submission` | `missing` | Integrate a real client or installer runtime guard before claiming native permission enforcement. |
| `native-client-telemetry` | `awaiting-submission` | `missing` | Install a real client against the native host and import production metadata-only events. |
### Provider Holdout
- readiness: `awaiting-submission`
- blocking reason: No real evidence submission has been provided yet.
- owner: operator with provider credentials
- template: `evidence/world_class/templates/provider-holdout.intake.json`
- submission: `evidence/world_class/submissions/provider-holdout.json`
#### Commands
- prepare_submission: `mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/provider-holdout.intake.json evidence/world_class/submissions/provider-holdout.json`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
#### Must Collect
- provenance_requirements:
- provider-backed model run
- observed timing
- observed token metadata
- success_checks:
- reports/output_execution_runs.json summary.model_executed_count > 0
- reports/output_execution_runs.json summary.timing_observed_count > 0
- reports/output_execution_runs.json summary.token_observed_count > 0
- reports/skill_os2_audit.json item provider-holdout status becomes pass
- evidence_artifacts:
- reports/output_execution_runs.json
- reports/output_execution_runs.md
- reports/skill_os2_audit.json
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/provider-holdout.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
- privacy_contract:
- Do not commit provider credentials or environment dumps.
- The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
### Human Adjudication
- readiness: `awaiting-submission`
- blocking reason: No real evidence submission has been provided yet.
- owner: human reviewer
- template: `evidence/world_class/templates/human-adjudication.intake.json`
- submission: `evidence/world_class/submissions/human-adjudication.json`
#### Commands
- prepare_submission: `mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/human-adjudication.intake.json evidence/world_class/submissions/human-adjudication.json`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
#### Must Collect
- provenance_requirements:
- real reviewer identity
- blind A/B decisions
- answer key unopened until decisions exist
- success_checks:
- reports/output_review_adjudication.json summary.pending_count == 0
- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
- reports/output_review_adjudication.json summary.invalid_decision_count == 0
- reports/skill_os2_audit.json item human-adjudication status becomes pass
- evidence_artifacts:
- reports/output_blind_review_pack.md
- reports/output_review_decisions.json
- reports/output_review_adjudication.json
- reports/output_review_adjudication.md
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/human-adjudication.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
- privacy_contract:
- Reviewer decisions should not include raw user data or private customer detail.
- Keep the answer key separate until after decisions are recorded.
### Native Permission Enforcement
- readiness: `awaiting-submission`
- blocking reason: No real evidence submission has been provided yet.
- owner: target client or installer integrator
- template: `evidence/world_class/templates/native-permission-enforcement.intake.json`
- submission: `evidence/world_class/submissions/native-permission-enforcement.json`
#### Commands
- prepare_submission: `mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-permission-enforcement.intake.json evidence/world_class/submissions/native-permission-enforcement.json`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
#### Must Collect
- provenance_requirements:
- real target or installer guard
- native enforcement flag
- residual risk retained for fallback targets
- success_checks:
- reports/runtime_permission_probes.json summary.native_enforcement_count > 0
- reports/runtime_permission_probes.json summary.failure_count == 0
- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
- evidence_artifacts:
- dist/targets/*/adapter.json
- reports/runtime_permission_probes.json
- reports/runtime_permission_probes.md
- security/permission_policy.json
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/native-permission-enforcement.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
- privacy_contract:
- Do not mark native_enforcement true for metadata-only fallbacks.
- Keep residual risks visible for targets that still rely on operator enforcement.
### Native Client Telemetry
- readiness: `awaiting-submission`
- blocking reason: No real evidence submission has been provided yet.
- owner: Browser/Chrome/IDE/provider client integrator
- template: `evidence/world_class/templates/native-client-telemetry.intake.json`
- submission: `evidence/world_class/submissions/native-client-telemetry.json`
#### Commands
- prepare_submission: `mkdir -p evidence/world_class/submissions && cp evidence/world_class/templates/native-client-telemetry.intake.json evidence/world_class/submissions/native-client-telemetry.json`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
#### Must Collect
- provenance_requirements:
- real external client source
- metadata-only event
- local-first import path
- success_checks:
- reports/adoption_drift_report.json summary.source_types.external > 0
- reports/adoption_drift_report.json summary.adoption_sample_count > 0
- reports/skill_os2_audit.json item native-client-telemetry status becomes pass
- evidence_artifacts:
- reports/adoption_drift_report.json
- reports/adoption_drift_report.md
- reports/telemetry_hook_recipes.json
- scripts/telemetry_native_host.py
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/native-client-telemetry.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
- privacy_contract:
- Telemetry must remain metadata-only and local-first.
- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
## Boundary
- Templates and planned work do not count as accepted evidence.
scripts/render_review_studio.py
+10 -2
View File
@@ -10,7 +10,7 @@ from review_studio_data import evidence_paths, insight_cards, load_review_data
from review_studio_formatting import registry_package_summary, render_kv_grid
from review_studio_gates import add_blockers_from_gate, build_gates, status_label, weighted_score
from review_studio_layout import render_review_nav, review_studio_css
from review_studio_world_class import render_world_class_evidence_entries
from review_studio_world_class import render_world_class_evidence_entries, render_world_class_intake_checklist
ROOT = Path(__file__).resolve().parent.parent
@@ -452,9 +452,11 @@ def render_html(report: dict[str, Any]) -> str:
waiver_summary = report["data"]["review_waivers"].get("summary", {})
world_class_ledger = report["data"].get("world_class_evidence_ledger", {})
world_class_summary = world_class_ledger.get("summary", {})
world_class_intake_summary = report["data"].get("world_class_evidence_intake", {}).get("summary", {})
world_class_intake = report["data"].get("world_class_evidence_intake", {})
world_class_intake_summary = world_class_intake.get("summary", {})
world_class_claim_summary = report["data"].get("world_class_claim_guard", {}).get("summary", {})
world_class_entries_html = render_world_class_evidence_entries(world_class_ledger)
world_class_intake_checklist_html = render_world_class_intake_checklist(world_class_intake)
annotation_summary = report["data"]["review_annotations"].get("summary", {})
annotation_caption = (
f"{annotation_summary.get('annotation_count', 0)} 条批注；"
@@ -782,6 +784,12 @@ def render_html(report: dict[str, Any]) -> str:
{world_class_entries_html}
</section>
<section>
<h2>提交清单</h2>
<p class="muted">每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。</p>
{world_class_intake_checklist_html}
</section>
<section class="twocol">
<div class="panel"><h2>蓝图覆盖</h2>{blueprint_panel}</div>
<div class="panel"><h2>覆盖边界</h2><p>蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。</p></div>
scripts/render_world_class_evidence_intake.py
+127
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import json
import shlex
from datetime import date
from pathlib import Path
from typing import Any
@@ -188,6 +189,87 @@ def find_entry(entries: list[dict[str, Any]], key: str) -> dict[str, Any] | None
return None
def first_by_key(items: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
by_key: dict[str, dict[str, Any]] = {}
for item in items:
key = str(item.get("evidence_key", ""))
if key and key not in by_key:
by_key[key] = item
return by_key
def shell_path(path: Path, root: Path) -> str:
return shlex.quote(rel_path(path, root))
def checklist_readiness(
entry: dict[str, Any],
template_result: dict[str, Any] | None,
submission_result: dict[str, Any] | None,
) -> tuple[str, str]:
if entry.get("status") == "accepted":
return "accepted", "Ledger already accepts this evidence item."
if template_result is None or template_result.get("status") != "pass":
return "fix-template", "The intake template is missing or invalid."
if submission_result is None:
return "awaiting-submission", "No real evidence submission has been provided yet."
if submission_result.get("status") == "pass":
return "ready-for-ledger-review", "Submission passes intake validation and is ready for ledger review."
return "fix-submission", "Submission exists but failed intake validation."
def build_operator_checklist(
skill_dir: Path,
ledger: dict[str, Any],
template_results: list[dict[str, Any]],
submission_results: list[dict[str, Any]],
submissions_dir: Path,
) -> list[dict[str, Any]]:
templates_by_key = first_by_key(template_results)
submissions_by_key = first_by_key(submission_results)
checklist = []
for entry in ledger.get("entries", []):
key = str(entry.get("key", ""))
template_path = skill_dir / "evidence" / "world_class" / "templates" / f"{key}.intake.json"
submission_path = submissions_dir / f"{key}.json"
template_result = templates_by_key.get(key)
submission_result = submissions_by_key.get(key)
readiness, blocking_reason = checklist_readiness(entry, template_result, submission_result)
submissions_dir_arg = shell_path(submissions_dir, skill_dir)
checklist.append(
{
"evidence_key": key,
"label": entry.get("label", key),
"category": entry.get("category", "external"),
"owner": entry.get("owner", "release reviewer"),
"readiness": readiness,
"blocking_reason": blocking_reason,
"template_status": template_result.get("status", "missing") if template_result else "missing",
"submission_status": submission_result.get("status", "missing") if submission_result else "missing",
"template_path": rel_path(template_path, skill_dir),
"submission_path": rel_path(submission_path, skill_dir),
"commands": {
"prepare_submission": (
f"mkdir -p {shell_path(submission_path.parent, skill_dir)} && "
f"cp {shell_path(template_path, skill_dir)} {shell_path(submission_path, skill_dir)}"
),
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {submissions_dir_arg}",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
},
"must_collect": {
"provenance_requirements": entry.get("provenance_requirements", []),
"success_checks": entry.get("success_checks", []),
"evidence_artifacts": entry.get("evidence_artifacts", []),
"privacy_contract": entry.get("privacy_contract", []),
},
"anti_overclaim": entry.get("anti_overclaim", {}),
"next_action": entry.get("next_action", ""),
}
)
return checklist
def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
ledger = build_ledger(skill_dir, generated_at)
entries = ledger.get("entries", [])
@@ -234,6 +316,8 @@ def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
invalid_submission_count = sum(1 for item in submission_results if item["status"] == "fail")
schema_exists = schema_path.exists()
intake_ready = schema_exists and template_pass_count == len(keys) and invalid_submission_count == 0
operator_checklist = build_operator_checklist(skill_dir, ledger, template_results, submission_results, submissions_dir)
ready_checklist_count = sum(1 for item in operator_checklist if item["readiness"] in {"accepted", "ready-for-ledger-review"})
return {
"schema_version": "1.0",
"ok": intake_ready,
@@ -247,6 +331,8 @@ def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
"submission_count": len(submission_results),
"valid_submission_count": valid_submission_count,
"invalid_submission_count": invalid_submission_count,
"operator_checklist_count": len(operator_checklist),
"operator_checklist_ready_count": ready_checklist_count,
"ready_for_external_collection": intake_ready,
"ready_for_ledger_review": valid_submission_count > 0 and invalid_submission_count == 0,
"ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True,
@@ -260,6 +346,7 @@ def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
"schema": rel_path(schema_path, skill_dir),
"templates": template_results,
"submissions": submission_results,
"operator_checklist": operator_checklist,
"source_ledger": {
"json": "reports/world_class_evidence_ledger.json",
"markdown": "reports/world_class_evidence_ledger.md",
@@ -284,6 +371,41 @@ def render_table(items: list[dict[str, Any]]) -> list[str]:
return lines
def render_operator_checklist(items: list[dict[str, Any]]) -> list[str]:
lines = [
"| Evidence | Readiness | Submission | Next action |",
"| --- | --- | --- | --- |",
]
if not items:
lines.append("| `none` | `accepted` | none | none |")
return lines
for item in items:
action = str(item.get("next_action", "")).replace("|", "\\|")
lines.append(
f"| `{item['evidence_key']}` | `{item['readiness']}` | `{item['submission_status']}` | {action} |"
)
for item in items:
lines.extend(["", f"### {item['label']}", ""])
lines.append(f"- readiness: `{item['readiness']}`")
lines.append(f"- blocking reason: {item['blocking_reason']}")
lines.append(f"- owner: {item['owner']}")
lines.append(f"- template: `{item['template_path']}`")
lines.append(f"- submission: `{item['submission_path']}`")
lines.extend(["", "#### Commands", ""])
commands = item.get("commands", {})
for label in ["prepare_submission", "validate_intake", "refresh_ledger", "guard_claim"]:
if commands.get(label):
lines.append(f"- {label}: `{commands[label]}`")
must_collect = item.get("must_collect", {})
lines.extend(["", "#### Must Collect", ""])
for label in ["provenance_requirements", "success_checks", "evidence_artifacts", "privacy_contract"]:
values = must_collect.get(label, [])
if values:
lines.append(f"- {label}:")
lines.extend(f" - {value}" for value in values)
return lines
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
@@ -298,6 +420,7 @@ def render_markdown(report: dict[str, Any]) -> str:
f"- templates: `{summary['template_pass_count']}` / `{summary['template_count']}`",
f"- submissions: `{summary['valid_submission_count']}` valid / `{summary['submission_count']}` total",
f"- invalid submissions: `{summary['invalid_submission_count']}`",
f"- operator checklist: `{summary['operator_checklist_ready_count']}` ready / `{summary['operator_checklist_count']}` total",
f"- ready for external collection: `{str(summary['ready_for_external_collection']).lower()}`",
f"- ready for ledger review: `{str(summary['ready_for_ledger_review']).lower()}`",
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
@@ -313,6 +436,10 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
*render_table(report["submissions"]),
"",
"## Operator Checklist",
"",
*render_operator_checklist(report["operator_checklist"]),
"",
"## Boundary",
"",
"- Templates and planned work do not count as accepted evidence.",
scripts/review_studio_layout.py
+72 -1
View File
@@ -253,6 +253,77 @@ def review_studio_css() -> str:
margin: 0;
padding-left: 18px;
}
.world-intake-grid {
display: grid;
grid-template-columns: minmax(0, 1fr);
gap: 16px;
margin-top: 16px;
}
.world-intake-card {
border: 1px solid var(--line);
border-radius: 8px;
padding: 18px;
background: #fff;
display: grid;
gap: 12px;
min-width: 0;
}
.world-intake-card.awaiting-submission,
.world-intake-card.fix-submission,
.world-intake-card.fix-template { border-left: 4px solid var(--warn); }
.world-intake-card.ready-for-ledger-review,
.world-intake-card.accepted { border-left: 4px solid var(--pass); }
.world-intake-card span,
.world-intake-card p,
.world-intake-card dd,
.world-intake-card li {
color: var(--muted);
font-size: 14px;
overflow-wrap: anywhere;
}
.world-intake-card dl {
display: grid;
grid-template-columns: 86px minmax(0, 1fr);
gap: 6px 10px;
margin: 0;
}
.world-intake-card dt { color: var(--ink); font-size: 14px; }
.world-intake-card dd { margin: 0; min-width: 0; }
.world-intake-steps {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
border-top: 1px solid var(--line);
padding-top: 12px;
}
.world-intake-steps h4 {
margin: 0 0 6px;
color: var(--ink);
font-size: 14px;
font-weight: 500;
}
.world-intake-steps ul {
margin: 0;
padding-left: 18px;
}
.world-intake-commands {
list-style: none;
padding: 0;
display: grid;
gap: 8px;
}
.world-intake-commands li {
display: grid;
gap: 2px;
min-width: 0;
padding: 8px 0 0;
border-top: 1px solid var(--line);
}
.world-intake-commands code {
overflow-wrap: anywhere;
white-space: normal;
display: block;
}
.action-card span,
.annotation-card span,
.action-card small,
@@ -302,7 +373,7 @@ def review_studio_css() -> str:
font-size: 13px;
}
@media (max-width: 980px) {
.metrics, .gates, .twocol, .actions-grid, .annotations-grid, .world-evidence-grid, .world-evidence-columns, .kv-grid { grid-template-columns: 1fr; }
.metrics, .gates, .twocol, .actions-grid, .annotations-grid, .world-evidence-grid, .world-evidence-columns, .world-intake-grid, .world-intake-steps, .kv-grid { grid-template-columns: 1fr; }
main { padding: 32px 18px 60px; }
nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }
nav a { flex: 0 0 auto; }
scripts/review_studio_world_class.py
+54
View File
@@ -50,3 +50,57 @@ def render_world_class_evidence_entries(ledger: dict[str, Any]) -> str:
"</article>"
)
return "<div class='world-evidence-grid'>" + "".join(cards) + "</div>"
def render_command_list(commands: dict[str, Any]) -> str:
if not commands:
return "<p class='muted'>暂无命令。</p>"
items = []
for label, command in commands.items():
if not command:
continue
items.append(
"<li>"
f"<span>{html.escape(str(label))}</span>"
f"<code>{html.escape(str(command))}</code>"
"</li>"
)
return "<ul class='world-intake-commands'>" + "".join(items) + "</ul>" if items else "<p class='muted'>暂无命令。</p>"
def render_world_class_intake_checklist(intake: dict[str, Any]) -> str:
items = intake.get("operator_checklist", []) if isinstance(intake, dict) else []
if not items:
return "<p class='muted'>当前没有 world-class 证据操作清单。</p>"
cards = []
for item in items:
readiness = str(item.get("readiness", "awaiting-submission"))
must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {}
cards.append(
"<article class='world-intake-card "
+ html.escape(readiness)
+ "'>"
f"<div><span>{html.escape(readiness)} · {html.escape(str(item.get('category', '')))}</span>"
f"<h3>{html.escape(str(item.get('label', item.get('evidence_key', 'evidence'))))}</h3></div>"
f"<p>{html.escape(str(item.get('blocking_reason', '')))}</p>"
f"<dl><dt>负责人</dt><dd>{html.escape(str(item.get('owner', '')))}</dd>"
f"<dt>模板</dt><dd><code>{html.escape(str(item.get('template_path', '')))}</code></dd>"
f"<dt>提交</dt><dd><code>{html.escape(str(item.get('submission_path', '')))}</code></dd>"
f"<dt>下一步</dt><dd>{html.escape(str(item.get('next_action', '')))}</dd></dl>"
"<div class='world-intake-steps'>"
"<div><h4>操作命令</h4>"
+ render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {})
+ "</div>"
"<div><h4>收集要求</h4>"
+ render_inline_list(must_collect.get("provenance_requirements", []), "暂无来源要求。")
+ "</div>"
"<div><h4>通过条件</h4>"
+ render_inline_list(must_collect.get("success_checks", []), "暂无通过条件。")
+ "</div>"
"<div><h4>隐私边界</h4>"
+ render_inline_list(must_collect.get("privacy_contract", []), "暂无隐私边界。")
+ "</div>"
"</div>"
"</article>"
)
return "<div class='world-intake-grid'>" + "".join(cards) + "</div>"
tests/verify_review_studio.py
+16
View File
@@ -410,7 +410,16 @@ def main() -> None:
assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4, full_payload["data"]["world_class_evidence_ledger"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["template_pass_count"] == 4, full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_count"] == 4, full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_ready_count"] == 0, full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["ready_to_claim_world_class"] is False, full_payload["data"]["world_class_evidence_intake"]
intake_checklist = full_payload["data"]["world_class_evidence_intake"]["operator_checklist"]
assert len(intake_checklist) == 4, intake_checklist
provider_checklist = next(item for item in intake_checklist if item["evidence_key"] == "provider-holdout")
assert provider_checklist["readiness"] == "awaiting-submission", provider_checklist
assert provider_checklist["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", provider_checklist
assert provider_checklist["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", provider_checklist
assert "provider-backed model run" in provider_checklist["must_collect"]["provenance_requirements"], provider_checklist
assert full_payload["data"]["world_class_claim_guard"]["summary"]["decision"] == "claim-guard-pass-evidence-pending", full_payload["data"]["world_class_claim_guard"]
assert full_payload["data"]["world_class_claim_guard"]["summary"]["violation_count"] == 0, full_payload["data"]["world_class_claim_guard"]
assert full_payload["data"]["world_class_claim_guard"]["summary"]["ledger_pending_count"] == 4, full_payload["data"]["world_class_claim_guard"]
@@ -478,6 +487,13 @@ def main() -> None:
assert "入口边界" in html, html
assert "声明守卫" in html, html
assert "声明边界" in html, html
assert "提交清单" in html, html
assert "world-intake-grid" in html, html
assert "操作命令" in html, html
assert "收集要求" in html, html
assert "通过条件" in html, html
assert "evidence/world_class/submissions/provider-holdout.json" in html, html
assert "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" in html, html
assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html
assert "reports/world_class_evidence_intake.md" in html, html
assert "reports/world_class_claim_guard.md" in html, html
tests/verify_world_class_evidence_intake.py
+29
View File
@@ -90,6 +90,8 @@ def main() -> None:
assert summary["submission_count"] == 0, summary
assert summary["valid_submission_count"] == 0, summary
assert summary["invalid_submission_count"] == 0, summary
assert summary["operator_checklist_count"] == 4, summary
assert summary["operator_checklist_ready_count"] == 0, summary
assert summary["ready_for_external_collection"] is True, summary
assert summary["ready_for_ledger_review"] is False, summary
assert summary["ready_to_claim_world_class"] is False, summary
@@ -101,9 +103,28 @@ def main() -> None:
"native-client-telemetry",
}, payload["templates"]
assert all(item["status"] == "pass" and item["template_only"] is True for item in payload["templates"]), payload["templates"]
checklist = {item["evidence_key"]: item for item in payload["operator_checklist"]}
assert set(checklist) == {
"provider-holdout",
"human-adjudication",
"native-permission-enforcement",
"native-client-telemetry",
}, checklist
assert checklist["provider-holdout"]["readiness"] == "awaiting-submission", checklist["provider-holdout"]
assert checklist["provider-holdout"]["template_path"] == "evidence/world_class/templates/provider-holdout.intake.json", checklist["provider-holdout"]
assert checklist["provider-holdout"]["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", checklist["provider-holdout"]
assert "cp evidence/world_class/templates/provider-holdout.intake.json" in checklist["provider-holdout"]["commands"]["prepare_submission"], checklist["provider-holdout"]
assert checklist["provider-holdout"]["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"]
assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"]
assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"]
assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"]
markdown = default_md.read_text(encoding="utf-8")
assert "World-Class Evidence Intake" in markdown, markdown
assert "ready to claim world-class: `false`" in markdown, markdown
assert "Operator Checklist" in markdown, markdown
assert "operator checklist: `0` ready / `4` total" in markdown, markdown
assert "`evidence/world_class/submissions/provider-holdout.json`" in markdown, markdown
assert "`python3 scripts/yao.py world-class-ledger .`" in markdown, markdown
assert "Templates and planned work do not count as accepted evidence." in markdown, markdown
valid_dir = TMP / "valid_submissions"
@@ -116,9 +137,14 @@ def main() -> None:
assert valid_payload["ok"] is True, valid_payload
assert valid_payload["summary"]["decision"] == "intake-ready-for-ledger-review", valid_payload["summary"]
assert valid_payload["summary"]["valid_submission_count"] == 1, valid_payload["summary"]
assert valid_payload["summary"]["operator_checklist_ready_count"] == 1, valid_payload["summary"]
assert valid_payload["summary"]["ready_for_ledger_review"] is True, valid_payload["summary"]
assert valid_payload["summary"]["ready_to_claim_world_class"] is False, valid_payload["summary"]
assert valid_payload["submissions"][0]["status"] == "pass", valid_payload["submissions"]
valid_checklist = {item["evidence_key"]: item for item in valid_payload["operator_checklist"]}
assert valid_checklist["provider-holdout"]["readiness"] == "ready-for-ledger-review", valid_checklist["provider-holdout"]
assert valid_checklist["provider-holdout"]["submission_path"].endswith("tests/tmp_world_class_evidence_intake/valid_submissions/provider-holdout.json"), valid_checklist["provider-holdout"]
assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["validate_intake"], valid_checklist["provider-holdout"]
invalid_dir = TMP / "invalid_submissions"
invalid_dir.mkdir()
@@ -130,8 +156,11 @@ def main() -> None:
assert invalid_payload["ok"] is False, invalid_payload
assert invalid_payload["summary"]["decision"] == "fix-intake", invalid_payload["summary"]
assert invalid_payload["summary"]["invalid_submission_count"] == 1, invalid_payload["summary"]
assert invalid_payload["summary"]["operator_checklist_ready_count"] == 0, invalid_payload["summary"]
assert any("raw content" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"]
assert any("attestation.real_external_or_human_evidence" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"]
invalid_checklist = {item["evidence_key"]: item for item in invalid_payload["operator_checklist"]}
assert invalid_checklist["provider-holdout"]["readiness"] == "fix-submission", invalid_checklist["provider-holdout"]
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
tests/verify_yao_cli.py
+7
View File
@@ -261,7 +261,14 @@ def main() -> None:
assert world_class_intake_result["ok"], world_class_intake_result
assert world_class_intake_result["payload"]["summary"]["decision"] == "awaiting-submissions", world_class_intake_result
assert world_class_intake_result["payload"]["summary"]["template_pass_count"] == 4, world_class_intake_result
assert world_class_intake_result["payload"]["summary"]["operator_checklist_count"] == 4, world_class_intake_result
assert world_class_intake_result["payload"]["summary"]["operator_checklist_ready_count"] == 0, world_class_intake_result
assert world_class_intake_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_intake_result
provider_checklist = next(
item for item in world_class_intake_result["payload"]["operator_checklist"] if item["evidence_key"] == "provider-holdout"
)
assert provider_checklist["readiness"] == "awaiting-submission", provider_checklist
assert "cp evidence/world_class/templates/provider-holdout.intake.json" in provider_checklist["commands"]["prepare_submission"], provider_checklist
assert world_class_ledger_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_ledger_result
world_class_claim_guard_result = run(