frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Include preflight in benchmark reproducibility

Browse Source
This commit is contained in:
yaojingang
2026-06-16 01:49:51 +08:00
parent dd33d55b62
commit d3fb78f4fe
17 changed files with 112 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1"
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1",
"archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e"
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/compiled_targets.json
+50 -40
View File
@@ -387,7 +387,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -446,6 +446,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -497,14 +498,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -526,7 +527,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -639,7 +640,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -698,6 +699,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -749,14 +751,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -776,7 +778,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -1252,7 +1254,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -1311,6 +1313,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1362,14 +1365,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1391,7 +1394,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -1504,7 +1507,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -1563,6 +1566,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1614,14 +1618,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1641,7 +1645,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -2117,7 +2121,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -2176,6 +2180,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2227,14 +2232,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2256,7 +2261,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -2362,7 +2367,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -2421,6 +2426,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2472,14 +2478,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2499,7 +2505,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -2966,7 +2972,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -3025,6 +3031,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3076,14 +3083,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3105,7 +3112,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -3211,7 +3218,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -3270,6 +3277,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3321,14 +3329,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3348,7 +3356,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -3815,7 +3823,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -3874,6 +3882,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3925,14 +3934,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3954,7 +3963,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
@@ -4064,7 +4073,7 @@
},
"file_write": {
"required": true,
"script_count": 69,
"script_count": 70,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
@@ -4123,6 +4132,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -4174,14 +4184,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 82,
"checked_count": 83,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 69,
"file_write_script_count": 70,
"subprocess_script_count": 10,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -4201,7 +4211,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 69,
"file_write": 70,
"subprocess": 10,
"interactive": 0
},
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.89,
"duration_ms": 30.29,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.42,
"duration_ms": 30.71,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.21,
"duration_ms": 31.61,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.52,
"duration_ms": 31.54,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.67,
"duration_ms": 30.87,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.97,
"duration_ms": 29.9,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.12,
"duration_ms": 31.08,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.11,
"duration_ms": 31.5,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.05,
"duration_ms": 31.33,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.87,
"duration_ms": 29.65,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 26.89 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.42 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 26.21 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.52 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.67 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.97 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.12 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.11 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.05 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.87 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 30.29 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 30.71 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 31.61 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.54 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 30.87 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.9 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.08 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.5 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.33 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 29.65 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"archive_entry_count": 623,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e`
- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1",
"archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e"
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1"
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1`
- Archive SHA256: `5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e`
- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d`
- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d`
- Install simulated: `True`
## Compatibility
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 200,
"package_sha256": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1"
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `200`
- Package SHA256: `909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1`
- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d`
## Failures
reports/skill-os-2-review.md
+1 -1
View File
@@ -130,7 +130,7 @@ Next move: add real client or installer permission enforcement integration.
| Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps |
| World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts |
| World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion |
| Benchmark Reproducibility | local reproducibility ready with `24` required artifacts, `0` missing artifacts, `22` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations |
| Benchmark Reproducibility | local reproducibility ready with `25` required artifacts, `0` missing artifacts, `23` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations |
| IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable |
| Context Budget | initial load `960/1000`, under the production budget |
| CI | `make ci-test` target count is `77` after the world-class preflight gate update |
reports/skill_os2_coverage.json
+1 -1
View File
@@ -245,7 +245,7 @@
"label": "Review Studio",
"status": "pass",
"objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.",
"current": "16 gates; decision review; warnings 3",
"current": "16 gates; decision review; warnings 4",
"command": "python3 scripts/yao.py review-studio .",
"test": "python3 tests/verify_review_studio.py",
"evidence": [
reports/skill_os2_coverage.md
+1 -1
View File
@@ -29,7 +29,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Trust Security | `pass` | 113 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
| Registry Distribution | `pass` | archive entries 623; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 4 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
## Recommended PR Coverage
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "5c411a7ea1891bf797cd9f7f2d7b02f6bde20055e383d17040725f7116e8583e"
"to": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "909d0f5bad8ede6cc24b6c733b614a36c178878447612aa97276ff05e5206ce1"
"to": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
}
]
},
scripts/render_benchmark_reproducibility.py
+6
View File
@@ -41,6 +41,7 @@ REQUIRED_ARTIFACTS = [
("world_class_evidence_plan", "reports/world_class_evidence_plan.json"),
("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"),
("world_class_evidence_intake", "reports/world_class_evidence_intake.json"),
("world_class_evidence_preflight", "reports/world_class_evidence_preflight.json"),
("world_class_submission_review", "reports/world_class_submission_review.json"),
("world_class_operator_runbook", "reports/world_class_operator_runbook.json"),
("world_class_operator_runbook_markdown", "reports/world_class_operator_runbook.md"),
@@ -134,6 +135,11 @@ REPRODUCTION_COMMANDS = [
"command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"evidence": "reports/world_class_evidence_intake.json",
},
{
"label": "world-class evidence preflight",
"command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
"evidence": "reports/world_class_evidence_preflight.json",
},
{
"label": "world-class submission review",
"command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
tests/verify_benchmark_reproducibility.py
+17
View File
@@ -40,6 +40,19 @@ def main() -> None:
text=True,
check=True,
)
subprocess.run(
[
sys.executable,
str(ROOT / "scripts" / "render_world_class_preflight.py"),
str(ROOT),
"--generated-at",
"2026-06-13",
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
subprocess.run(
[
sys.executable,
@@ -163,6 +176,7 @@ def main() -> None:
assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts
assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts
assert artifacts["reports/world_class_evidence_intake.json"]["exists"], artifacts
assert artifacts["reports/world_class_evidence_preflight.json"]["exists"], artifacts
assert artifacts["reports/world_class_submission_review.json"]["exists"], artifacts
assert artifacts["reports/world_class_operator_runbook.json"]["exists"], artifacts
assert artifacts["reports/world_class_operator_runbook.md"]["exists"], artifacts
@@ -172,6 +186,7 @@ def main() -> None:
assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload
@@ -190,7 +205,9 @@ def main() -> None:
assert "## Release Lock" in markdown, markdown
assert "## Evidence Bundle" in markdown, markdown
assert "reports/benchmark_methodology.md" in markdown, markdown
assert "reports/world_class_evidence_preflight.json" in markdown, markdown
assert "reports/world_class_operator_runbook.html" in markdown, markdown
assert "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" in markdown, markdown
assert "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" in markdown, markdown
assert "python3 scripts/yao.py evidence-consistency ." in markdown, markdown
assert "make ci-test" in markdown, markdown