frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add world-class operator runbook

Browse Source
This commit is contained in:
yaojingang
2026-06-14 08:06:56 +08:00
parent 08347d5c2c
commit 5d1d3bbd30
58 changed files with 2029 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+2
View File
@@ -45,6 +45,7 @@ python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED
python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_overview.py .
python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json
@@ -75,6 +76,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept
- `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete.
- `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence.
- `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state.
- `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness.
- `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable.
- `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation.
- `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers.
Makefile
+6 -3
View File
@@ -2,7 +2,7 @@ PYTHON ?= python3
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -112,6 +112,9 @@ world-class-intake-check:
world-class-submission-review-check:
$(PYTHON) tests/verify_world_class_submission_review.py
world-class-runbook-check:
$(PYTHON) tests/verify_world_class_operator_runbook.py
world-class-claim-guard-check:
$(PYTHON) tests/verify_world_class_claim_guard.py
@@ -232,11 +235,11 @@ sync-local-install: package-check
sync-active-install: package-check
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
ci-test:
$(PYTHON) scripts/ci_test.py
clean:
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
find . -type d -name __pycache__ -prune -exec rm -rf {} +
registry/index.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"schema_version": "2.0",
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"package_count": 1,
"packages": [
{
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
}
]
}
registry/packages/yao-meta-skill.json
+4 -4
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771",
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b"
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e"
},
"compatibility": {
"openai": "pass",
@@ -48,10 +48,10 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
"generated_at": "2026-06-14"
}
reports/adoption_drift_report.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13T23:43:44Z",
"generated_at": "2026-06-14T00:05:43Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
reports/architecture_maintainability.json
+12 -12
View File
@@ -4,11 +4,11 @@
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"python_file_count": 151,
"script_file_count": 93,
"test_file_count": 58,
"python_file_count": 153,
"script_file_count": 94,
"test_file_count": 59,
"internal_module_count": 19,
"cli_script_count": 75,
"cli_script_count": 76,
"command_handler_count": 34,
"warn_line_threshold": 900,
"block_line_threshold": 1500,
@@ -27,14 +27,14 @@
},
{
"path": "tests/verify_yao_cli.py",
"lines": 899,
"lines": 888,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/yao.py",
"lines": 867,
"lines": 874,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
@@ -47,16 +47,16 @@
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/compile_skill.py",
"lines": 734,
"kind": "cli-script",
"path": "scripts/yao_cli_parser.py",
"lines": 741,
"kind": "internal-module",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/yao_cli_parser.py",
"lines": 729,
"kind": "internal-module",
"path": "scripts/compile_skill.py",
"lines": 734,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
reports/architecture_maintainability.md
+7 -7
View File
@@ -5,11 +5,11 @@ Generated at: `2026-06-13`
## Summary
- decision: `pass`
- python files: `151`
- scripts: `93`
- tests: `58`
- python files: `153`
- scripts: `94`
- tests: `59`
- internal modules: `19`
- CLI scripts: `75`
- CLI scripts: `76`
- Yao CLI command handlers: `34`
- largest file lines: `899`
- hotspots: `0`
@@ -26,11 +26,11 @@ No file-size hotspots found.
| File | Lines | Kind | Severity |
| --- | ---: | --- | --- |
| `scripts/render_review_studio.py` | `899` | `cli-script` | `pass` |
| `tests/verify_yao_cli.py` | `899` | `test` | `pass` |
| `scripts/yao.py` | `867` | `cli-script` | `pass` |
| `tests/verify_yao_cli.py` | `888` | `test` | `pass` |
| `scripts/yao.py` | `874` | `cli-script` | `pass` |
| `scripts/skill_report_model.py` | `792` | `internal-module` | `pass` |
| `scripts/yao_cli_parser.py` | `741` | `internal-module` | `pass` |
| `scripts/compile_skill.py` | `734` | `cli-script` | `pass` |
| `scripts/yao_cli_parser.py` | `729` | `internal-module` | `pass` |
| `scripts/optimize_description.py` | `723` | `cli-script` | `pass` |
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
reports/benchmark_reproducibility.json
+27 -27
View File
@@ -3,24 +3,24 @@
"ok": true,
"generated_at": "2026-06-14",
"skill_dir": ".",
"commit": "6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130",
"commit": "08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1",
"git_status": {
"available": true,
"dirty": true,
"changed_file_count": 49,
"changed_file_count": 61,
"sample": [
" M registry/index.json",
" M registry/packages/yao-meta-skill.json",
" M reports/adoption_drift_report.json",
"M AGENTS.md",
"M Makefile",
"M registry/index.json",
"M registry/packages/yao-meta-skill.json",
"MM reports/adoption_drift_report.json",
" M reports/adoption_drift_report.md",
" M reports/architecture_maintainability.json",
" M reports/architecture_maintainability.md",
" M reports/benchmark_reproducibility.json",
" M reports/benchmark_reproducibility.md",
" M reports/compiled_targets.json",
" M reports/context_budget.json",
" M reports/install_simulation.json",
" M reports/install_simulation.md"
"MM reports/architecture_maintainability.json",
"MM reports/architecture_maintainability.md",
"MM reports/benchmark_reproducibility.json",
"MM reports/benchmark_reproducibility.md",
"MM reports/compiled_targets.json",
"MM reports/context_budget.json"
]
},
"summary": {
@@ -42,7 +42,7 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 49
"changed_file_count": 61
},
"methodology": {
"path": "reports/benchmark_methodology.md",
@@ -115,8 +115,8 @@
"label": "output_execution",
"path": "reports/output_execution_runs.json",
"exists": true,
"bytes": 7967,
"sha256": "b0953427365ce84f89454531d5819a2ce6f494875661fbbb3be33b012a0d8f13"
"bytes": 7965,
"sha256": "99b2e3f03710fdcee8fe66a476bc589f4e22a379bfcc1cd5561de49c184286e8"
},
{
"label": "blind_review",
@@ -150,43 +150,43 @@
"label": "trust_report",
"path": "reports/security_trust_report.json",
"exists": true,
"bytes": 95873,
"sha256": "cb599bf68524be22e7df189f1f09deac404bda493a3013c74be209d7ef7efae4"
"bytes": 97005,
"sha256": "252c580028df6ac2404b53cc10bb607fa82b98ef17965cb7354c4731f6c6b08e"
},
{
"label": "python_compatibility",
"path": "reports/python_compatibility.json",
"exists": true,
"bytes": 20093,
"sha256": "e5dd973d0b5c78c5e40d52727939134b8167d742cb90bea2141744c0f7d5fa33"
"bytes": 20367,
"sha256": "a8d14087863459534fadac760a64c213c9474413e92046a9ac2e772d4174d347"
},
{
"label": "registry_audit",
"path": "reports/registry_audit.json",
"exists": true,
"bytes": 3183,
"sha256": "2ff87aeb6f3dc7f0614d33d291faad1a55da2ade3ba98014bee5143dfb8f4806"
"sha256": "6527b723a4bc367477e1090afbf7f1653f20d33fed0e0691725b7ee30896848b"
},
{
"label": "package_verification",
"path": "reports/package_verification.json",
"exists": true,
"bytes": 19325,
"sha256": "a5dac0ea2b6365c97d0b72ad695ed0ebbd5ac472745e20c1a98f695b26bdd827"
"sha256": "6bf19c9e29803ad8668914489d22b9e306ad0ad092c91c179c3aaffe81eed2c6"
},
{
"label": "install_simulation",
"path": "reports/install_simulation.json",
"exists": true,
"bytes": 8604,
"sha256": "eebfba42d71e4082d3c801fd126e59c6401b3f8c00a00625c70eb82f927ebd65"
"bytes": 8758,
"sha256": "270a28b82dc49d80ba9c5a82f912cc960e50a5ac42f022c6a2c345e4e84fbb36"
},
{
"label": "skill_os2_audit",
"path": "reports/skill_os2_audit.json",
"exists": true,
"bytes": 14309,
"sha256": "78bcbcbc54711e63a6e34059c30748599a7e71d434182b78c307af6d2553cc81"
"sha256": "cad5296cd500a40cb0d83897d327b3542f31c1ce6bfe7eaa387ea2e88d229aa1"
},
{
"label": "world_class_evidence_plan",
@@ -213,8 +213,8 @@
"label": "world_class_claim_guard",
"path": "reports/world_class_claim_guard.json",
"exists": true,
"bytes": 8267,
"sha256": "79c5b29f0ff639189aa65f3487afab13150e95c7fa7a6a5b59309b1e87873c25"
"bytes": 8463,
"sha256": "250d616b028cf046e2033f9e2c5648c8d95c110d0d8990b35e4d481e14cfc558"
}
],
"missing_artifacts": [],
reports/benchmark_reproducibility.md
+10 -10
View File
@@ -1,7 +1,7 @@
# Benchmark Reproducibility
Generated at: `2026-06-14`
Commit: `6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130`
Commit: `08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1`
Working tree dirty at generation: `true`
## Summary
@@ -16,7 +16,7 @@ Working tree dirty at generation: `true`
- provider evidence complete: `false`
- human review complete: `false`
- world-class ready: `false`
- changed files at generation: `49`
- changed files at generation: `61`
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
@@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
| output_execution | `reports/output_execution_runs.json` | present | `b0953427365c` |
| output_execution | `reports/output_execution_runs.json` | present | `99b2e3f03710` |
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
| trust_report | `reports/security_trust_report.json` | present | `cb599bf68524` |
| python_compatibility | `reports/python_compatibility.json` | present | `e5dd973d0b5c` |
| registry_audit | `reports/registry_audit.json` | present | `2ff87aeb6f3d` |
| package_verification | `reports/package_verification.json` | present | `a5dac0ea2b63` |
| install_simulation | `reports/install_simulation.json` | present | `eebfba42d71e` |
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `78bcbcbc5471` |
| trust_report | `reports/security_trust_report.json` | present | `252c580028df` |
| python_compatibility | `reports/python_compatibility.json` | present | `a8d140878634` |
| registry_audit | `reports/registry_audit.json` | present | `6527b723a4bc` |
| package_verification | `reports/package_verification.json` | present | `6bf19c9e2980` |
| install_simulation | `reports/install_simulation.json` | present | `270a28b82dc4` |
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `cad5296cd500` |
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `02e49b66159d` |
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0b52d0d1528b` |
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `5fbfcd35ac6a` |
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `79c5b29f0ff6` |
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` |
## Reproduction Commands
reports/compiled_targets.json
+61 -51
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"target_count": 5,
@@ -205,6 +205,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -223,8 +224,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
@@ -383,7 +383,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -437,6 +437,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -486,14 +487,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -515,7 +516,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -628,7 +629,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -682,6 +683,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -731,14 +733,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -758,7 +760,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -1052,6 +1054,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -1070,8 +1073,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
@@ -1230,7 +1232,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -1284,6 +1286,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1333,14 +1336,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1362,7 +1365,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -1475,7 +1478,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -1529,6 +1532,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1578,14 +1582,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1605,7 +1609,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -1899,6 +1903,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -1917,8 +1922,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
@@ -2077,7 +2081,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2131,6 +2135,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2180,14 +2185,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2209,7 +2214,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -2315,7 +2320,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2369,6 +2374,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2418,14 +2424,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2445,7 +2451,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -2730,6 +2736,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -2748,8 +2755,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
@@ -2908,7 +2914,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2962,6 +2968,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3011,14 +3018,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3040,7 +3047,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -3146,7 +3153,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -3200,6 +3207,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3249,14 +3257,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3276,7 +3284,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -3561,6 +3569,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -3579,8 +3588,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
@@ -3739,7 +3747,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -3793,6 +3801,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3842,14 +3851,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3871,7 +3880,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
@@ -3981,7 +3990,7 @@
},
"file_write": {
"required": true,
"script_count": 63,
"script_count": 64,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -4035,6 +4044,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -4084,14 +4094,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 76,
"checked_count": 77,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 63,
"file_write_script_count": 64,
"subprocess_script_count": 9,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -4111,7 +4121,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 63,
"file_write": 64,
"subprocess": 9,
"interactive": 0
},
reports/context_budget.json
+3 -3
View File
@@ -6,10 +6,10 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 1157142,
"other_text_tokens": 1175718,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 1157893,
"relevant_file_count": 491,
"estimated_total_text_tokens": 1176469,
"relevant_file_count": 498,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 137.7
reports/install_simulation.json
+10 -10
View File
@@ -1,14 +1,14 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"package_dir": "tests/tmp_review_studio/dist",
"install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill",
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
"package_dir": "dist",
"install_root": "dist/install-simulation/simulate-yao-meta-skill",
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
"summary": {
"archive_present": true,
"archive_entry_count": 568,
"archive_entry_count": 573,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
@@ -26,7 +26,7 @@
{
"id": "archive-present",
"status": "pass",
"detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip"
"detail": "Package archive exists: dist/yao-meta-skill.zip"
},
{
"id": "archive-safe-paths",
@@ -272,9 +272,9 @@
"failures": [],
"warnings": [],
"artifacts": {
"archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip",
"package_manifest": "tests/tmp_review_studio/dist/manifest.json",
"json": "tests/tmp_review_studio/install_simulation.json",
"markdown": "tests/tmp_review_studio/install_simulation.md"
"archive": "dist/yao-meta-skill.zip",
"package_manifest": "dist/manifest.json",
"json": "reports/install_simulation.json",
"markdown": "reports/install_simulation.md"
}
}
reports/install_simulation.md
+2 -2
View File
@@ -1,7 +1,7 @@
# Install Simulation
- OK: `True`
- Package directory: `tests/tmp_review_studio/dist`
- Package directory: `dist`
- Archive extracted: `True`
- Entrypoint loaded: `True`
- Manifest loaded: `True`
@@ -16,7 +16,7 @@
| Check | Status | Detail |
| --- | --- | --- |
| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip |
| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip |
| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries |
| `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill |
| `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable |
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.97,
"duration_ms": 26.47,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.97,
"duration_ms": 25.78,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.26,
"duration_ms": 25.73,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.71,
"duration_ms": 29.11,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.89,
"duration_ms": 29.2,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.93,
"duration_ms": 28.81,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.5,
"duration_ms": 25.65,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.37,
"duration_ms": 26.22,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.2,
"duration_ms": 25.43,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.27,
"duration_ms": 25.88,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 27.97 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.97 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.26 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.71 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.89 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.93 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.5 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.37 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.2 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.27 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 26.47 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 25.78 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 25.73 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.11 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.2 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.81 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.65 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.22 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.43 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.88 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+2 -2
View File
@@ -8,8 +8,8 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b",
"archive_entry_count": 568,
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e",
"archive_entry_count": 573,
"failure_count": 0,
"warning_count": 0
},
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b`
- Archive SHA256: `de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e`
- Failures: `0`
- Warnings: `0`
reports/python_compatibility.json
+13 -1
View File
@@ -5,7 +5,7 @@
"root": ".",
"summary": {
"target_python": "3.11",
"file_count": 154,
"file_count": 156,
"issue_count": 0,
"syntax_error_count": 0,
"fstring_311_violation_count": 0,
@@ -400,6 +400,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "scripts/render_world_class_operator_runbook.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "scripts/render_world_class_submission_review.py",
"ok": true,
@@ -934,6 +940,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "tests/verify_world_class_operator_runbook.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "tests/verify_world_class_submission_review.py",
"ok": true,
reports/python_compatibility.md
+1 -1
View File
@@ -6,7 +6,7 @@ Generated at: `2026-06-13`
- decision: `pass`
- target python: `3.11`
- files scanned: `154`
- files scanned: `156`
- issues: `0`
- syntax errors: `0`
- f-string 3.11 violations: `0`
reports/registry_audit.json
+6 -6
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771",
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b"
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e"
},
"compatibility": {
"openai": "pass",
@@ -53,16 +53,16 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
"generated_at": "2026-06-14"
},
"index": {
"schema_version": "2.0",
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"package_count": 1,
"packages": [
{
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771`
- Archive SHA256: `cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b`
- Package SHA256: `97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179`
- Archive SHA256: `de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e`
- Install simulated: `True`
## Compatibility
reports/review-studio.html
+9 -9
View File
File diff suppressed because one or more lines are too long
reports/review-studio.json
+508 -131
View File
File diff suppressed because it is too large Load Diff
reports/security_trust_report.json
+34 -9
View File
@@ -2,8 +2,8 @@
"ok": true,
"skill_dir": ".",
"summary": {
"scanned_files": 177,
"script_count": 93,
"scanned_files": 178,
"script_count": 94,
"internal_module_count": 17,
"secret_findings": 0,
"dependency_files": [
@@ -12,18 +12,18 @@
"network_script_count": 3,
"network_policy_covered_count": 3,
"network_policy_missing_count": 0,
"file_write_script_count": 63,
"file_write_script_count": 64,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 76,
"help_smoke_checked_count": 77,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 177,
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"package_hash_file_count": 178,
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
},
"failures": [],
"warnings": [],
@@ -886,6 +886,20 @@
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/render_world_class_operator_runbook.py",
"interface": "cli",
"interface_declared": true,
"interface_reason": "Renders an operator runbook for collecting pending world-class evidence without accepting evidence.",
"has_argparse": true,
"has_main_guard": true,
"uses_input": false,
"uses_network": false,
"uses_file_write": true,
"uses_subprocess": false,
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/render_world_class_submission_review.py",
"interface": "cli",
@@ -1379,9 +1393,9 @@
"help_smoke": {
"enabled": true,
"timeout_seconds": 5.0,
"candidate_count": 76,
"checked_count": 76,
"passed_count": 76,
"candidate_count": 77,
"checked_count": 77,
"passed_count": 77,
"failed_count": 0,
"skipped_count": 17,
"failed_scripts": [],
@@ -1986,6 +2000,16 @@
"stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ",
"stderr_excerpt": ""
},
{
"path": "scripts/render_world_class_operator_runbook.py",
"command": "python3 scripts/render_world_class_operator_runbook.py --help",
"returncode": 0,
"timed_out": false,
"passed": true,
"has_help_text": true,
"stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ",
"stderr_excerpt": ""
},
{
"path": "scripts/render_world_class_submission_review.py",
"command": "python3 scripts/render_world_class_submission_review.py --help",
@@ -2322,6 +2346,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
reports/security_trust_report.md
+9 -8
View File
@@ -1,22 +1,22 @@
# Security Trust Report
- OK: `True`
- Scanned files: `177`
- Scripts: `93`
- Scanned files: `178`
- Scripts: `94`
- Internal script modules: `17`
- Secret findings: `0`
- Network-capable scripts: `3`
- Network policy covered scripts: `3`
- Network policy missing scripts: `0`
- File-write scripts: `63`
- File-write scripts: `64`
- Permission approvals: `3 / 3`
- Permission approval gaps: `0`
- CLI help smoke checked: `76`
- CLI help smoke checked: `77`
- CLI help smoke failures: `0`
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `177`
- Package SHA256: `7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771`
- Package hash files: `178`
- Package SHA256: `97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179`
## Failures
@@ -54,8 +54,8 @@
- Enabled: `True`
- Timeout seconds: `5.0`
- Checked scripts: `76`
- Passed scripts: `76`
- Checked scripts: `77`
- Passed scripts: `77`
- Failed scripts: `none`
## Script Surface
@@ -122,6 +122,7 @@
| scripts/render_world_class_evidence_intake.py | cli | True | True | True | False | False | True | False | Validates world-class human and external evidence intake packets before ledger review. |
| scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. |
| scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_world_class_operator_runbook.py | cli | True | True | True | False | False | True | False | Renders an operator runbook for collecting pending world-class evidence without accepting evidence. |
| scripts/render_world_class_submission_review.py | cli | True | True | True | False | False | True | False | Renders a read-only review queue for world-class evidence submissions before ledger acceptance. |
| scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/review_studio_data.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to load Review Studio source reports and metric cards. |
reports/skill-overview.html
+1 -1
View File
@@ -771,7 +771,7 @@
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
</div>
<div class="two-col">
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.8 75.2" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.1 27.9" stroke-dashoffset="-27.9" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">357项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.6 75.4" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.3 27.7" stroke-dashoffset="-27.7" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">361项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<table>
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
reports/skill-overview.json
+23 -23
View File
@@ -437,7 +437,7 @@
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 93
"file_count": 94
},
{
"path": "evals",
@@ -449,10 +449,10 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 200
"file_count": 203
}
],
"file_count": 357,
"file_count": 361,
"folder_count": 4,
"distribution": [
{
@@ -477,7 +477,7 @@
},
{
"label": "scripts",
"value": 93
"value": 94
},
{
"label": "evals",
@@ -485,7 +485,7 @@
},
{
"label": "reports",
"value": 200
"value": 203
}
]
},
@@ -606,7 +606,7 @@
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 93
"file_count": 94
},
{
"path": "evals",
@@ -618,7 +618,7 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 200
"file_count": 203
}
],
"strengths": [
@@ -935,9 +935,9 @@
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 32
"changed_file_count": 61
},
"commit": "6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130",
"commit": "08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1",
"missing_artifacts": [],
"limitations": [
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
@@ -1020,8 +1020,8 @@
"failures": []
},
"trust_security": {
"scanned_files": 177,
"script_count": 93,
"scanned_files": 178,
"script_count": 94,
"internal_module_count": 17,
"secret_findings": 0,
"dependency_files": [
@@ -1030,18 +1030,18 @@
"network_script_count": 3,
"network_policy_covered_count": 3,
"network_policy_missing_count": 0,
"file_write_script_count": 63,
"file_write_script_count": 64,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 76,
"help_smoke_checked_count": 77,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 177,
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"package_hash_file_count": 178,
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
},
"skill_atlas": {
"skill_count": 12,
@@ -1079,8 +1079,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771",
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b"
"package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e"
},
"compatibility": {
"openai": "pass",
@@ -1111,7 +1111,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b",
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1127,8 +1127,8 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b",
"archive_entry_count": 568,
"archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e",
"archive_entry_count": 573,
"failure_count": 0,
"warning_count": 0
},
@@ -1139,7 +1139,7 @@
"ok": true,
"summary": {
"archive_present": true,
"archive_entry_count": 568,
"archive_entry_count": 573,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
@@ -1206,12 +1206,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b"
"to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
}
]
},
reports/skill_atlas.json
+1
View File
@@ -124,6 +124,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
reports/skill_os2_audit.json
+2 -2
View File
@@ -197,7 +197,7 @@
"key": "trust-security",
"label": "Trust Security",
"status": "pass",
"current": "secrets 0; scripts 93; help failures 0",
"current": "secrets 0; scripts 94; help failures 0",
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
"evidence": [
{
@@ -285,7 +285,7 @@
"key": "registry-distribution",
"label": "Registry Distribution",
"status": "pass",
"current": "zip entries 568; install failures 0; permission failures 0",
"current": "zip entries 573; install failures 0; permission failures 0",
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
"evidence": [
{
reports/skill_os2_audit.md
+2 -2
View File
@@ -23,11 +23,11 @@ Generated at: `2026-06-13`
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
| Benchmark Reproducibility | pass | artifacts 20; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
| Trust Security | pass | secrets 0; scripts 93; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Trust Security | pass | secrets 0; scripts 94; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. |
| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
| Registry Distribution | pass | zip entries 568; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Registry Distribution | pass | zip entries 573; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
reports/skill_os2_coverage.json
+2 -2
View File
@@ -122,7 +122,7 @@
"label": "Trust Security",
"status": "pass",
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
"current": "93 scripts; secrets 0; help failures 0",
"current": "94 scripts; secrets 0; help failures 0",
"command": "python3 scripts/yao.py trust .",
"test": "python3 tests/verify_trust_check.py",
"evidence": [
@@ -196,7 +196,7 @@
"label": "Registry Distribution",
"status": "pass",
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
"current": "archive entries 568; install failures 0",
"current": "archive entries 573; install failures 0",
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
"test": "python3 tests/verify_registry_audit.py",
"evidence": [
reports/skill_os2_coverage.md
+2 -2
View File
@@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` |
| Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` |
| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` |
| Trust Security | `pass` | 93 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Trust Security | `pass` | 94 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
| Registry Distribution | `pass` | archive entries 568; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
| Registry Distribution | `pass` | archive entries 573; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
reports/upgrade_check.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"previous": {
"name": "yao-meta-skill",
"version": "1.0.0",
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b"
"to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771"
"to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179"
}
]
},
@@ -90,7 +90,7 @@
"artifacts": {
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
"current_package": "reports/registry_audit.json",
"json": "tests/tmp_review_studio/upgrade_check.json",
"markdown": "tests/tmp_review_studio/upgrade_check.md"
"json": "reports/upgrade_check.json",
"markdown": "reports/upgrade_check.md"
}
}
reports/world_class_claim_guard.json
+10 -2
View File
@@ -1,12 +1,12 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"ledger_ready_to_claim_world_class": false,
"ledger_pending_count": 4,
"claim_surface_count": 71,
"claim_surface_count": 73,
"violation_count": 0,
"overclaim_guard_active": true,
"decision": "claim-guard-pass-evidence-pending"
@@ -315,6 +315,14 @@
"path": "reports/world_class_evidence_plan.md",
"violation_count": 0
},
{
"path": "reports/world_class_operator_runbook.html",
"violation_count": 0
},
{
"path": "reports/world_class_operator_runbook.md",
"violation_count": 0
},
{
"path": "reports/world_class_submission_review.md",
"violation_count": 0
reports/world_class_claim_guard.md
+2 -2
View File
@@ -1,13 +1,13 @@
# World-Class Claim Guard
Generated at: `2026-06-13`
Generated at: `2026-06-14`
## Summary
- decision: `claim-guard-pass-evidence-pending`
- ledger ready to claim world-class: `false`
- ledger pending evidence: `4`
- claim surfaces scanned: `71`
- claim surfaces scanned: `73`
- violations: `0`
- overclaim guard active: `true`
reports/world_class_evidence_intake.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"schema_present": true,
reports/world_class_evidence_intake.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Intake
Generated at: `2026-06-13`
Generated at: `2026-06-14`
## Summary
reports/world_class_evidence_ledger.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"ledger_entry_count": 4,
reports/world_class_evidence_ledger.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Ledger
Generated at: `2026-06-13`
Generated at: `2026-06-14`
## Summary
reports/world_class_evidence_plan.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"audit_decision": "continue-iteration",
reports/world_class_evidence_plan.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Plan
Generated at: `2026-06-13`
Generated at: `2026-06-14`
## Summary
reports/world_class_operator_runbook.html
+118
View File
@@ -0,0 +1,118 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>World-Class Operator Runbook</title>
<style>
:root { --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --pass:#1f6f43; }
* { box-sizing:border-box; }
body { margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }
.topbar { position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); }
.topbar-inner { max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:16px; align-items:center; }
.brand, a { color:var(--ink); }
.links { display:flex; gap:14px; flex-wrap:wrap; }
.links a { text-decoration:none; }
.shell { max-width:1180px; margin:0 auto; padding:36px 24px 72px; }
.hero { border-bottom:1px solid var(--line); padding:32px 0 28px; }
.eyebrow { color:var(--ink); font-size:12px; text-transform:uppercase; font-weight:700; letter-spacing:0; }
h1 { margin:8px 0 12px; color:var(--ink); font-size:56px; line-height:1.04; letter-spacing:0; }
h2, h3, h4 { color:var(--ink); letter-spacing:0; }
h2 { margin:0 0 14px; font-size:30px; }
h3 { margin:4px 0 10px; font-size:22px; }
h4 { margin:0 0 8px; font-size:16px; }
.lede { max-width:800px; color:var(--muted); font-size:20px; }
.stats { display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin-top:26px; }
.stats article, .panel, .item-card { border:1px solid var(--line); border-radius:8px; background:#fff; }
.stats article { padding:16px; }
.stats span, .item-card span, .muted { color:var(--muted); }
.stats strong { display:block; color:var(--ink); font-size:34px; line-height:1; }
.section { padding:32px 0; border-bottom:1px solid var(--line); }
.item-grid { display:grid; grid-template-columns:1fr; gap:18px; }
.item-card { padding:20px; border-left:4px solid var(--warn); }
.item-card.ready-for-ledger-review, .item-card.accepted { border-left-color:var(--pass); }
dl { display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }
dt { color:var(--ink); }
dd { margin:0; min-width:0; overflow-wrap:anywhere; }
code { font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:13px; overflow-wrap:anywhere; }
ul, ol { padding-left:20px; }
.commands { list-style:none; padding:0; margin:0; display:grid; gap:10px; }
.commands li { padding:12px; background:var(--soft); border-radius:8px; }
.commands span { display:block; color:var(--ink); font-weight:700; margin-bottom:4px; }
.mini-grid { display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; }
.mini-grid section, .panel { background:var(--soft); border-radius:8px; padding:14px; min-width:0; }
.mini-grid li, .panel li { overflow-wrap:anywhere; }
@media (max-width:820px) { .stats, .mini-grid { grid-template-columns:1fr; } h1 { font-size:38px; } .topbar-inner { align-items:flex-start; flex-direction:column; } }
</style>
</head>
<body>
<nav class="topbar"><div class="topbar-inner"><span class="brand">World-Class Runbook</span><div class="links"><a href="#fast-path">Fast Path</a><a href="#items">Evidence</a><a href="#boundary">Boundary</a></div></div></nav>
<main class="shell">
<section class="hero">
<span class="eyebrow">Evidence Operations</span>
<h1>World-Class Operator Runbook</h1>
<p class="lede">A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.</p>
<div class="stats"><article><span>Pending</span><strong>4</strong></article><article><span>Awaiting</span><strong>4</strong></article><article><span>Ready</span><strong></strong></article><article><span>Invalid</span><strong></strong></article></div>
</section>
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid"><article class="item-card awaiting-submission">
<header><span>external · awaiting-submission</span><h3>Provider Holdout</h3></header>
<p>No evidence packet has been submitted for review.</p>
<dl>
<dt>Owner</dt><dd>operator with provider credentials</dd>
<dt>Ledger</dt><dd><code>pending</code></dd>
<dt>Submission</dt><dd><code>evidence/world_class/submissions/provider-holdout.json</code></dd>
</dl>
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger .</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
<div class="mini-grid">
<section><h4>Must Collect</h4><ul><li>provider-backed model run</li><li>observed timing</li><li>observed token metadata</li></ul></section>
<section><h4>Success Checks</h4><ul><li>reports/output_execution_runs.json summary.model_executed_count &gt; 0</li><li>reports/output_execution_runs.json summary.timing_observed_count &gt; 0</li><li>reports/output_execution_runs.json summary.token_observed_count &gt; 0</li><li>reports/skill_os2_audit.json item provider-holdout status becomes pass</li></ul></section>
<section><h4>Privacy</h4><ul><li>Do not commit provider credentials or environment dumps.</li><li>The output execution report records output hashes and aggregate run metadata, not raw provider prompts.</li></ul></section>
</div>
</article><article class="item-card awaiting-submission">
<header><span>human · awaiting-submission</span><h3>Human Adjudication</h3></header>
<p>No evidence packet has been submitted for review.</p>
<dl>
<dt>Owner</dt><dd>human reviewer</dd>
<dt>Ledger</dt><dd><code>pending</code></dd>
<dt>Submission</dt><dd><code>evidence/world_class/submissions/human-adjudication.json</code></dd>
</dl>
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger .</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
<div class="mini-grid">
<section><h4>Must Collect</h4><ul><li>real reviewer identity</li><li>blind A/B decisions</li><li>answer key unopened until decisions exist</li></ul></section>
<section><h4>Success Checks</h4><ul><li>reports/output_review_adjudication.json summary.pending_count == 0</li><li>reports/output_review_adjudication.json summary.judgment_count == summary.pair_count</li><li>reports/output_review_adjudication.json summary.invalid_decision_count == 0</li><li>reports/skill_os2_audit.json item human-adjudication status becomes pass</li></ul></section>
<section><h4>Privacy</h4><ul><li>Reviewer decisions should not include raw user data or private customer detail.</li><li>Keep the answer key separate until after decisions are recorded.</li></ul></section>
</div>
</article><article class="item-card awaiting-submission">
<header><span>external · awaiting-submission</span><h3>Native Permission Enforcement</h3></header>
<p>No evidence packet has been submitted for review.</p>
<dl>
<dt>Owner</dt><dd>target client or installer integrator</dd>
<dt>Ledger</dt><dd><code>pending</code></dd>
<dt>Submission</dt><dd><code>evidence/world_class/submissions/native-permission-enforcement.json</code></dd>
</dl>
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger .</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
<div class="mini-grid">
<section><h4>Must Collect</h4><ul><li>real target client or external installer runtime guard</li><li>native enforcement flag or externally accepted guard proof</li><li>residual risk retained for fallback targets</li></ul></section>
<section><h4>Success Checks</h4><ul><li>reports/runtime_permission_probes.json summary.native_enforcement_count &gt; 0</li><li>reports/runtime_permission_probes.json summary.failure_count == 0</li><li>reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence</li><li>reports/skill_os2_audit.json item native-permission-enforcement status becomes pass</li></ul></section>
<section><h4>Privacy</h4><ul><li>Do not mark native_enforcement true for metadata-only fallbacks.</li><li>Keep residual risks visible for targets that still rely on operator enforcement.</li></ul></section>
</div>
</article><article class="item-card awaiting-submission">
<header><span>external · awaiting-submission</span><h3>Native Client Telemetry</h3></header>
<p>No evidence packet has been submitted for review.</p>
<dl>
<dt>Owner</dt><dd>Browser/Chrome/IDE/provider client integrator</dd>
<dt>Ledger</dt><dd><code>pending</code></dd>
<dt>Submission</dt><dd><code>evidence/world_class/submissions/native-client-telemetry.json</code></dd>
</dl>
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger .</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
<div class="mini-grid">
<section><h4>Must Collect</h4><ul><li>real external client source</li><li>metadata-only event</li><li>local-first import path</li></ul></section>
<section><h4>Success Checks</h4><ul><li>reports/adoption_drift_report.json summary.source_types.external &gt; 0</li><li>reports/adoption_drift_report.json summary.adoption_sample_count &gt; 0</li><li>reports/skill_os2_audit.json item native-client-telemetry status becomes pass</li></ul></section>
<section><h4>Privacy</h4><ul><li>Telemetry must remain metadata-only and local-first.</li><li>Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.</li></ul></section>
</div>
</article></div></section>
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for ledger review, not accepted evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
</main>
</body>
</html>
reports/world_class_operator_runbook.json
+319
View File
@@ -0,0 +1,319 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"evidence_item_count": 4,
"pending_count": 4,
"accepted_count": 0,
"awaiting_submission_count": 4,
"ready_for_ledger_review_count": 0,
"valid_packet_source_incomplete_count": 0,
"invalid_submission_count": 0,
"ready_to_claim_world_class": false,
"runbook_counts_as_completion": false,
"decision": "collect-evidence"
},
"submissions": {
"directory": "evidence/world_class/submissions",
"runbook_counts_submission_as_completion": false
},
"items": [
{
"evidence_key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"ledger_status": "pending",
"intake_readiness": "awaiting-submission",
"review_state": "awaiting-submission",
"source_accepted": false,
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"current": "model-executed 0; token-observed 0",
"blocking_reason": "No evidence packet has been submitted for review.",
"submission_path": "evidence/world_class/submissions/provider-holdout.json",
"template_path": "evidence/world_class/templates/provider-holdout.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
]
},
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/provider-holdout.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"observed_state": {
"model_executed_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"accepted": false
},
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/provider-holdout.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
}
},
{
"evidence_key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"ledger_status": "pending",
"intake_readiness": "awaiting-submission",
"review_state": "awaiting-submission",
"source_accepted": false,
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
"current": "0/5 decisions; pending 5",
"blocking_reason": "No evidence packet has been submitted for review.",
"submission_path": "evidence/world_class/submissions/human-adjudication.json",
"template_path": "evidence/world_class/templates/human-adjudication.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
]
},
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_kit.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/human-adjudication.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"observed_state": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"accepted": false
},
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/human-adjudication.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
}
},
{
"evidence_key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"ledger_status": "pending",
"intake_readiness": "awaiting-submission",
"review_state": "awaiting-submission",
"source_accepted": false,
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
"current": "native-enforced targets 0; installer-enforced targets 4",
"blocking_reason": "No evidence packet has been submitted for review.",
"submission_path": "evidence/world_class/submissions/native-permission-enforcement.json",
"template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real target client or external installer runtime guard",
"native enforcement flag or externally accepted guard proof",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
]
},
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"reports/install_simulation.json",
"reports/install_simulation.md",
"security/permission_policy.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-permission-enforcement.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"observed_state": {
"native_enforcement_count": 0,
"metadata_fallback_count": 4,
"installer_enforcement_pass_count": 4,
"installer_permission_failure_count": 0,
"installer_enforcement_ready": true,
"residual_risk_count": 4,
"failure_count": 0,
"accepted": false
},
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/native-permission-enforcement.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
}
},
{
"evidence_key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"ledger_status": "pending",
"intake_readiness": "awaiting-submission",
"review_state": "awaiting-submission",
"source_accepted": false,
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
"current": "external source events 0; adoption samples 1",
"blocking_reason": "No evidence packet has been submitted for review.",
"submission_path": "evidence/world_class/submissions/native-client-telemetry.json",
"template_path": "evidence/world_class/templates/native-client-telemetry.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger .",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"must_collect": {
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
]
},
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-client-telemetry.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"observed_state": {
"external_source_events": 0,
"adoption_sample_count": 1,
"raw_content_allowed": false,
"risk_band": "low",
"accepted": false
},
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/native-client-telemetry.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
}
}
],
"source_reports": {
"ledger": "reports/world_class_evidence_ledger.json",
"intake": "reports/world_class_evidence_intake.json",
"submission_review": "reports/world_class_submission_review.json",
"claim_guard": "reports/world_class_claim_guard.json"
},
"artifacts": {
"json": "reports/world_class_operator_runbook.json",
"markdown": "reports/world_class_operator_runbook.md",
"html": "reports/world_class_operator_runbook.html"
}
}
reports/world_class_operator_runbook.md
+214
View File
@@ -0,0 +1,214 @@
# World-Class Operator Runbook
Generated at: `2026-06-14`
## Summary
- decision: `collect-evidence`
- ready to claim world-class: `false`
- runbook counts as completion: `false`
- evidence items: `4`
- pending: `4`
- awaiting submission: `4`
- ready for ledger review: `0`
This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.
## Fast Path
1. Run the real external or human work for one evidence item.
2. Generate the matching submission draft.
3. Replace template-only fields with aggregate evidence and provenance.
4. Validate intake, review the queue, refresh the ledger, then run the claim guard.
## Evidence Items
| Evidence | Ledger | Intake | Review | Owner |
| --- | --- | --- | --- | --- |
| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | operator with provider credentials |
| `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | human reviewer |
| `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | target client or installer integrator |
| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | Browser/Chrome/IDE/provider client integrator |
## Provider Holdout
- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.
- blocking reason: No evidence packet has been submitted for review.
- submission: `evidence/world_class/submissions/provider-holdout.json`
- template: `evidence/world_class/templates/provider-holdout.intake.json`
### Commands
- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
### Must Collect
- provider-backed model run
- observed timing
- observed token metadata
### Success Checks
- reports/output_execution_runs.json summary.model_executed_count > 0
- reports/output_execution_runs.json summary.timing_observed_count > 0
- reports/output_execution_runs.json summary.token_observed_count > 0
- reports/skill_os2_audit.json item provider-holdout status becomes pass
### Privacy Contract
- Do not commit provider credentials or environment dumps.
- The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
### Evidence Artifacts
- reports/output_execution_runs.json
- reports/output_execution_runs.md
- reports/skill_os2_audit.json
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/provider-holdout.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
## Human Adjudication
- objective: Record real blind A/B reviewer decisions before claiming human output review completion.
- blocking reason: No evidence packet has been submitted for review.
- submission: `evidence/world_class/submissions/human-adjudication.json`
- template: `evidence/world_class/templates/human-adjudication.intake.json`
### Commands
- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
### Must Collect
- real reviewer identity
- blind A/B decisions
- answer key unopened until decisions exist
### Success Checks
- reports/output_review_adjudication.json summary.pending_count == 0
- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
- reports/output_review_adjudication.json summary.invalid_decision_count == 0
- reports/skill_os2_audit.json item human-adjudication status becomes pass
### Privacy Contract
- Reviewer decisions should not include raw user data or private customer detail.
- Keep the answer key separate until after decisions are recorded.
### Evidence Artifacts
- reports/output_blind_review_pack.md
- reports/output_review_kit.md
- reports/output_review_decisions.json
- reports/output_review_adjudication.json
- reports/output_review_adjudication.md
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/human-adjudication.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
## Native Permission Enforcement
- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.
- blocking reason: No evidence packet has been submitted for review.
- submission: `evidence/world_class/submissions/native-permission-enforcement.json`
- template: `evidence/world_class/templates/native-permission-enforcement.intake.json`
### Commands
- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
### Must Collect
- real target client or external installer runtime guard
- native enforcement flag or externally accepted guard proof
- residual risk retained for fallback targets
### Success Checks
- reports/runtime_permission_probes.json summary.native_enforcement_count > 0
- reports/runtime_permission_probes.json summary.failure_count == 0
- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
### Privacy Contract
- Do not mark native_enforcement true for metadata-only fallbacks.
- Keep residual risks visible for targets that still rely on operator enforcement.
### Evidence Artifacts
- dist/targets/*/adapter.json
- reports/runtime_permission_probes.json
- reports/runtime_permission_probes.md
- reports/install_simulation.json
- reports/install_simulation.md
- security/permission_policy.json
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/native-permission-enforcement.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
## Native Client Telemetry
- objective: Import production metadata-only events from a real external client into the local drift loop.
- blocking reason: No evidence packet has been submitted for review.
- submission: `evidence/world_class/submissions/native-client-telemetry.json`
- template: `evidence/world_class/templates/native-client-telemetry.intake.json`
### Commands
- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions`
- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`
- refresh_ledger: `python3 scripts/yao.py world-class-ledger .`
- guard_claim: `python3 scripts/yao.py world-class-claim-guard .`
### Must Collect
- real external client source
- metadata-only event
- local-first import path
### Success Checks
- reports/adoption_drift_report.json summary.source_types.external > 0
- reports/adoption_drift_report.json summary.adoption_sample_count > 0
- reports/skill_os2_audit.json item native-client-telemetry status becomes pass
### Privacy Contract
- Telemetry must remain metadata-only and local-first.
- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
### Evidence Artifacts
- reports/adoption_drift_report.json
- reports/adoption_drift_report.md
- reports/telemetry_hook_recipes.json
- scripts/telemetry_native_host.py
- evidence/world_class/intake.schema.json
- evidence/world_class/templates/native-client-telemetry.intake.json
- reports/world_class_evidence_intake.json
- reports/world_class_evidence_intake.md
## Boundary
- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
- Valid intake means ready for ledger review, not accepted evidence.
- The world-class ledger and claim guard remain the source of truth.
reports/world_class_submission_review.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"generated_at": "2026-06-14",
"skill_dir": ".",
"summary": {
"review_item_count": 4,
reports/world_class_submission_review.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Submission Review
Generated at: `2026-06-13`
Generated at: `2026-06-14`
## Summary
scripts/ci_test.py
+1
View File
@@ -58,6 +58,7 @@ DEFAULT_TARGETS = [
"world-class-ledger-check",
"world-class-intake-check",
"world-class-submission-review-check",
"world-class-runbook-check",
"world-class-claim-guard-check",
"benchmark-reproducibility-check",
"feedback-check",
scripts/render_review_studio.py
+2 -2
View File
@@ -263,7 +263,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = {
"world-class-evidence": {
"summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。",
"why": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。",
"source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md",
"source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md",
"source_paths": [
{"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]},
{"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]},
@@ -278,7 +278,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = {
{"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]},
{"path": "reports/adoption_drift_report.md", "label": "adoption drift", "kind": "report", "patterns": ["# Adoption"]},
],
"verification": "python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .",
"verification": "python3 scripts/yao.py world-class-runbook . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .",
},
"registry-audit": {
"summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。",
scripts/render_world_class_operator_runbook.py
+349
View File
@@ -0,0 +1,349 @@
#!/usr/bin/env python3
import argparse
import html
import json
from datetime import date
from pathlib import Path
from typing import Any
from render_world_class_evidence_intake import build_intake
from render_world_class_evidence_ledger import build_ledger
from render_world_class_submission_review import build_submission_review
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Renders an operator runbook for collecting pending world-class evidence without accepting evidence."
def rel_path(path: Path, root: Path) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def html_text(value: Any) -> str:
return html.escape(str(value or ""), quote=True)
def by_key(items: list[dict[str, Any]], key_name: str) -> dict[str, dict[str, Any]]:
return {str(item.get(key_name, "")): item for item in items if item.get(key_name)}
def output_path(path: str, skill_dir: Path) -> str:
candidate = Path(path)
if candidate.is_absolute():
return rel_path(candidate, skill_dir)
return path
def build_runbook_item(
entry: dict[str, Any],
checklist: dict[str, Any],
review_item: dict[str, Any],
) -> dict[str, Any]:
commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}
must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {}
return {
"evidence_key": entry.get("key", ""),
"label": entry.get("label", entry.get("key", "")),
"category": entry.get("category", "external"),
"owner": entry.get("owner", "release reviewer"),
"ledger_status": entry.get("status", "pending"),
"intake_readiness": checklist.get("readiness", "missing"),
"review_state": review_item.get("review_state", "missing"),
"source_accepted": review_item.get("source_accepted") is True,
"objective": entry.get("objective", ""),
"current": entry.get("current", ""),
"blocking_reason": review_item.get("blocking_reason") or checklist.get("blocking_reason") or entry.get("next_action", ""),
"submission_path": checklist.get("submission_path") or entry.get("submission_state", {}).get("path", ""),
"template_path": checklist.get("template_path", ""),
"commands": {
"prepare_submission": commands.get("prepare_submission", ""),
"validate_intake": commands.get("validate_intake", ""),
"review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": commands.get("refresh_ledger", "python3 scripts/yao.py world-class-ledger ."),
"guard_claim": commands.get("guard_claim", "python3 scripts/yao.py world-class-claim-guard ."),
},
"must_collect": {
"provenance_requirements": must_collect.get("provenance_requirements", entry.get("provenance_requirements", [])),
"success_checks": must_collect.get("success_checks", entry.get("success_checks", [])),
"privacy_contract": must_collect.get("privacy_contract", entry.get("privacy_contract", [])),
},
"evidence_artifacts": entry.get("evidence_artifacts", []),
"observed_state": entry.get("observed_state", {}),
"submission_state": entry.get("submission_state", {}),
"anti_overclaim": entry.get("anti_overclaim", {}),
}
def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir)
intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir)
review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir)
checklist_by_key = by_key(intake.get("operator_checklist", []), "evidence_key")
review_by_key = by_key(review.get("items", []), "evidence_key")
items = [
build_runbook_item(entry, checklist_by_key.get(str(entry.get("key", "")), {}), review_by_key.get(str(entry.get("key", "")), {}))
for entry in ledger.get("entries", [])
]
summary = ledger.get("summary", {})
review_summary = review.get("summary", {})
return {
"schema_version": "1.0",
"ok": True,
"generated_at": generated_at,
"skill_dir": rel_path(skill_dir, ROOT),
"summary": {
"evidence_item_count": len(items),
"pending_count": summary.get("pending_count", 0),
"accepted_count": summary.get("accepted_count", 0),
"awaiting_submission_count": review_summary.get("awaiting_submission_count", 0),
"ready_for_ledger_review_count": review_summary.get("ready_for_ledger_review_count", 0),
"valid_packet_source_incomplete_count": review_summary.get("valid_packet_source_incomplete_count", 0),
"invalid_submission_count": review_summary.get("invalid_submission_count", 0),
"ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True,
"runbook_counts_as_completion": False,
"decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence",
},
"submissions": {
"directory": rel_path(submissions_dir, skill_dir),
"runbook_counts_submission_as_completion": False,
},
"items": items,
"source_reports": {
"ledger": "reports/world_class_evidence_ledger.json",
"intake": "reports/world_class_evidence_intake.json",
"submission_review": "reports/world_class_submission_review.json",
"claim_guard": "reports/world_class_claim_guard.json",
},
"artifacts": {
"json": "reports/world_class_operator_runbook.json",
"markdown": "reports/world_class_operator_runbook.md",
"html": "reports/world_class_operator_runbook.html",
},
}
def list_lines(values: list[Any], empty: str) -> list[str]:
if not values:
return [f"- {empty}"]
return [f"- {value}" for value in values]
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# World-Class Operator Runbook",
"",
f"Generated at: `{report['generated_at']}`",
"",
"## Summary",
"",
f"- decision: `{summary['decision']}`",
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
f"- runbook counts as completion: `{str(summary['runbook_counts_as_completion']).lower()}`",
f"- evidence items: `{summary['evidence_item_count']}`",
f"- pending: `{summary['pending_count']}`",
f"- awaiting submission: `{summary['awaiting_submission_count']}`",
f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`",
"",
"This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.",
"",
"## Fast Path",
"",
"1. Run the real external or human work for one evidence item.",
"2. Generate the matching submission draft.",
"3. Replace template-only fields with aggregate evidence and provenance.",
"4. Validate intake, review the queue, refresh the ledger, then run the claim guard.",
"",
"## Evidence Items",
"",
"| Evidence | Ledger | Intake | Review | Owner |",
"| --- | --- | --- | --- | --- |",
]
for item in report["items"]:
lines.append(
f"| `{item['evidence_key']}` | `{item['ledger_status']}` | `{item['intake_readiness']}` | "
f"`{item['review_state']}` | {item['owner']} |"
)
for item in report["items"]:
lines.extend(
[
"",
f"## {item['label']}",
"",
f"- objective: {item['objective']}",
f"- blocking reason: {item['blocking_reason']}",
f"- submission: `{item['submission_path'] or 'missing'}`",
f"- template: `{item['template_path'] or 'missing'}`",
"",
"### Commands",
"",
]
)
for label, command in item["commands"].items():
lines.append(f"- {label}: `{command}`")
lines.extend(["", "### Must Collect", ""])
lines.extend(list_lines(item["must_collect"].get("provenance_requirements", []), "No provenance requirements listed."))
lines.extend(["", "### Success Checks", ""])
lines.extend(list_lines(item["must_collect"].get("success_checks", []), "No success checks listed."))
lines.extend(["", "### Privacy Contract", ""])
lines.extend(list_lines(item["must_collect"].get("privacy_contract", []), "No privacy contract listed."))
lines.extend(["", "### Evidence Artifacts", ""])
lines.extend(list_lines(item.get("evidence_artifacts", []), "No evidence artifacts listed."))
lines.extend(
[
"",
"## Boundary",
"",
"- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.",
"- Valid intake means ready for ledger review, not accepted evidence.",
"- The world-class ledger and claim guard remain the source of truth.",
]
)
return "\n".join(lines).rstrip() + "\n"
def html_list(values: list[Any], empty: str) -> str:
if not values:
return f"<li>{html_text(empty)}</li>"
return "".join(f"<li>{html_text(value)}</li>" for value in values)
def render_html_item(item: dict[str, Any]) -> str:
commands = "".join(
f"<li><span>{html_text(label.replace('_', ' '))}</span><code>{html_text(command)}</code></li>"
for label, command in item["commands"].items()
)
must_collect = item["must_collect"]
return f"""
<article class="item-card {html_text(item['review_state'])}">
<header><span>{html_text(item['category'])} · {html_text(item['review_state'])}</span><h3>{html_text(item['label'])}</h3></header>
<p>{html_text(item['blocking_reason'])}</p>
<dl>
<dt>Owner</dt><dd>{html_text(item['owner'])}</dd>
<dt>Ledger</dt><dd><code>{html_text(item['ledger_status'])}</code></dd>
<dt>Submission</dt><dd><code>{html_text(item['submission_path'])}</code></dd>
</dl>
<section><h4>Commands</h4><ul class="commands">{commands}</ul></section>
<div class="mini-grid">
<section><h4>Must Collect</h4><ul>{html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}</ul></section>
<section><h4>Success Checks</h4><ul>{html_list(must_collect.get('success_checks', []), 'No success checks listed.')}</ul></section>
<section><h4>Privacy</h4><ul>{html_list(must_collect.get('privacy_contract', []), 'No privacy contract listed.')}</ul></section>
</div>
</article>
""".strip()
def render_html(report: dict[str, Any]) -> str:
summary = report["summary"]
stats = [
("Pending", summary["pending_count"]),
("Awaiting", summary["awaiting_submission_count"]),
("Ready", summary["ready_for_ledger_review_count"]),
("Invalid", summary["invalid_submission_count"]),
]
stat_html = "".join(f"<article><span>{html_text(label)}</span><strong>{html_text(value)}</strong></article>" for label, value in stats)
item_html = "".join(render_html_item(item) for item in report["items"])
return f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>World-Class Operator Runbook</title>
<style>
:root {{ --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --pass:#1f6f43; }}
* {{ box-sizing:border-box; }}
body {{ margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }}
.topbar {{ position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); }}
.topbar-inner {{ max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:16px; align-items:center; }}
.brand, a {{ color:var(--ink); }}
.links {{ display:flex; gap:14px; flex-wrap:wrap; }}
.links a {{ text-decoration:none; }}
.shell {{ max-width:1180px; margin:0 auto; padding:36px 24px 72px; }}
.hero {{ border-bottom:1px solid var(--line); padding:32px 0 28px; }}
.eyebrow {{ color:var(--ink); font-size:12px; text-transform:uppercase; font-weight:700; letter-spacing:0; }}
h1 {{ margin:8px 0 12px; color:var(--ink); font-size:56px; line-height:1.04; letter-spacing:0; }}
h2, h3, h4 {{ color:var(--ink); letter-spacing:0; }}
h2 {{ margin:0 0 14px; font-size:30px; }}
h3 {{ margin:4px 0 10px; font-size:22px; }}
h4 {{ margin:0 0 8px; font-size:16px; }}
.lede {{ max-width:800px; color:var(--muted); font-size:20px; }}
.stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin-top:26px; }}
.stats article, .panel, .item-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }}
.stats article {{ padding:16px; }}
.stats span, .item-card span, .muted {{ color:var(--muted); }}
.stats strong {{ display:block; color:var(--ink); font-size:34px; line-height:1; }}
.section {{ padding:32px 0; border-bottom:1px solid var(--line); }}
.item-grid {{ display:grid; grid-template-columns:1fr; gap:18px; }}
.item-card {{ padding:20px; border-left:4px solid var(--warn); }}
.item-card.ready-for-ledger-review, .item-card.accepted {{ border-left-color:var(--pass); }}
dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }}
dt {{ color:var(--ink); }}
dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }}
code {{ font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:13px; overflow-wrap:anywhere; }}
ul, ol {{ padding-left:20px; }}
.commands {{ list-style:none; padding:0; margin:0; display:grid; gap:10px; }}
.commands li {{ padding:12px; background:var(--soft); border-radius:8px; }}
.commands span {{ display:block; color:var(--ink); font-weight:700; margin-bottom:4px; }}
.mini-grid {{ display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; }}
.mini-grid section, .panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }}
.mini-grid li, .panel li {{ overflow-wrap:anywhere; }}
@media (max-width:820px) {{ .stats, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }}
</style>
</head>
<body>
<nav class="topbar"><div class="topbar-inner"><span class="brand">World-Class Runbook</span><div class="links"><a href="#fast-path">Fast Path</a><a href="#items">Evidence</a><a href="#boundary">Boundary</a></div></div></nav>
<main class="shell">
<section class="hero">
<span class="eyebrow">Evidence Operations</span>
<h1>World-Class Operator Runbook</h1>
<p class="lede">A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.</p>
<div class="stats">{stat_html}</div>
</section>
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid">{item_html}</div></section>
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for ledger review, not accepted evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
</main>
</body>
</html>
"""
def main() -> None:
parser = argparse.ArgumentParser(description="Render an operator runbook for collecting pending world-class evidence.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--submissions-dir")
parser.add_argument("--output-json", default="reports/world_class_operator_runbook.json")
parser.add_argument("--output-md", default="reports/world_class_operator_runbook.md")
parser.add_argument("--output-html", default="reports/world_class_operator_runbook.html")
parser.add_argument("--generated-at", default=date.today().isoformat())
args = parser.parse_args()
skill_dir = Path(args.skill_dir).resolve()
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
report = build_operator_runbook(skill_dir, args.generated_at, submissions_dir=submissions_dir)
outputs = {
"json": Path(args.output_json),
"markdown": Path(args.output_md),
"html": Path(args.output_html),
}
for key, path in outputs.items():
if not path.is_absolute():
path = skill_dir / path
path.parent.mkdir(parents=True, exist_ok=True)
report["artifacts"][key] = output_path(str(path), skill_dir)
if key == "json":
path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
elif key == "markdown":
path.write_text(render_markdown(report), encoding="utf-8")
else:
path.write_text(render_html(report), encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
scripts/review_studio_data.py
+3
View File
@@ -84,6 +84,8 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.md",
"world_class_evidence_intake": "reports/world_class_evidence_intake.md",
"world_class_submission_review": "reports/world_class_submission_review.md",
"world_class_operator_runbook": "reports/world_class_operator_runbook.md",
"world_class_operator_runbook_html": "reports/world_class_operator_runbook.html",
"world_class_claim_guard": "reports/world_class_claim_guard.md",
"registry_audit": "reports/registry_audit.md",
"package_verification": "reports/package_verification.md",
@@ -123,6 +125,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
"world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"),
"world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"),
"world_class_submission_review": load_json(reports / "world_class_submission_review.json"),
"world_class_operator_runbook": load_json(reports / "world_class_operator_runbook.json"),
"world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"),
"registry": load_json(reports / "registry_audit.json"),
"package_verification": load_json(reports / "package_verification.json"),
scripts/yao.py
+7
View File
@@ -34,6 +34,7 @@ from yao_cli_report_commands import (
command_world_class_evidence,
command_world_class_intake,
command_world_class_ledger,
command_world_class_runbook,
command_world_class_submission_kit,
command_world_class_submission_review,
)
@@ -176,6 +177,8 @@ def command_report(args: argparse.Namespace) -> int:
run_script("render_world_class_evidence_plan.py", [str(ROOT)]),
run_script("render_world_class_evidence_ledger.py", [str(ROOT)]),
run_script("render_world_class_evidence_intake.py", [str(ROOT)]),
run_script("render_world_class_submission_review.py", [str(ROOT)]),
run_script("render_world_class_operator_runbook.py", [str(ROOT)]),
run_script("render_world_class_claim_guard.py", [str(ROOT)]),
run_script("render_benchmark_reproducibility.py", [str(ROOT)]),
run_script("render_skill_os2_coverage.py", [str(ROOT)]),
@@ -212,6 +215,8 @@ def command_report(args: argparse.Namespace) -> int:
"world_class_evidence_plan": "reports/world_class_evidence_plan.json",
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.json",
"world_class_evidence_intake": "reports/world_class_evidence_intake.json",
"world_class_submission_review": "reports/world_class_submission_review.json",
"world_class_operator_runbook": "reports/world_class_operator_runbook.json",
"world_class_claim_guard": "reports/world_class_claim_guard.json",
"benchmark_reproducibility": "reports/benchmark_reproducibility.json",
"skill_os2_coverage": "reports/skill_os2_coverage.json",
@@ -741,6 +746,8 @@ def command_workspace_flow(args: argparse.Namespace) -> int:
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_intake.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_submission_review.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_operator_runbook.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_claim_guard.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_skill_os2_coverage.py", [str(ROOT)])},
scripts/yao_cli_parser.py
+12
View File
@@ -231,6 +231,18 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
world_class_submission_review_cmd.add_argument("--generated-at")
world_class_submission_review_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_review"))
world_class_runbook_cmd = subparsers.add_parser(
"world-class-runbook",
help="Render an operator runbook for pending world-class evidence.",
)
world_class_runbook_cmd.add_argument("skill_dir", nargs="?", default=".")
world_class_runbook_cmd.add_argument("--submissions-dir")
world_class_runbook_cmd.add_argument("--output-json")
world_class_runbook_cmd.add_argument("--output-md")
world_class_runbook_cmd.add_argument("--output-html")
world_class_runbook_cmd.add_argument("--generated-at")
world_class_runbook_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_runbook"))
world_class_claim_guard_cmd = subparsers.add_parser(
"world-class-claim-guard",
help="Scan public claim surfaces for premature world-class completion claims.",
scripts/yao_cli_report_commands.py
+8
View File
@@ -100,6 +100,14 @@ def command_world_class_submission_review(args: argparse.Namespace) -> int:
return emit_result(run_script("render_world_class_submission_review.py", cmd))
def command_world_class_runbook(args: argparse.Namespace) -> int:
cmd = [resolved_skill_dir(args)]
if args.submissions_dir:
cmd.extend(["--submissions-dir", args.submissions_dir])
append_outputs(cmd, args, generated_at=True)
return emit_result(run_script("render_world_class_operator_runbook.py", cmd))
def command_world_class_claim_guard(args: argparse.Namespace) -> int:
cmd = [resolved_skill_dir(args)]
for surface in args.claim_surface:
skill-ir/examples/yao-meta-skill.json
+2 -2
View File
@@ -168,6 +168,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
@@ -186,8 +187,7 @@
"scripts/skill_report_charts.py",
"scripts/skill_report_layout.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py"
"scripts/skill_report_model.py"
],
"assets": [
"templates/basic_skill.md.j2",
skill_atlas/catalog.json
+1
View File
@@ -80,6 +80,7 @@
"scripts/render_world_class_evidence_intake.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_data.py",
tests/verify_review_studio.py
+11 -1
View File
@@ -245,6 +245,7 @@ def main() -> None:
"render_world_class_evidence_ledger.py",
"render_world_class_evidence_intake.py",
"render_world_class_submission_review.py",
"render_world_class_operator_runbook.py",
"render_world_class_claim_guard.py",
"render_skill_os2_coverage.py",
]:
@@ -379,6 +380,10 @@ def main() -> None:
assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_submission_review.md").exists():
assert full_payload["evidence_paths"]["world_class_submission_review"] == "reports/world_class_submission_review.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_operator_runbook.md").exists():
assert full_payload["evidence_paths"]["world_class_operator_runbook"] == "reports/world_class_operator_runbook.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_operator_runbook.html").exists():
assert full_payload["evidence_paths"]["world_class_operator_runbook_html"] == "reports/world_class_operator_runbook.html", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_claim_guard.md").exists():
assert full_payload["evidence_paths"]["world_class_claim_guard"] == "reports/world_class_claim_guard.md", full_payload["evidence_paths"]
assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"]
@@ -420,12 +425,16 @@ def main() -> None:
"reports/skill_os2_audit.md",
}, world_class_action
assert all(item["exists"] for item in world_class_action["source_refs"]), world_class_action
assert "world-class-submission-review" in world_class_action["verification_command"], world_class_action
assert "world-class-runbook" in world_class_action["verification_command"], world_class_action
assert "reports/world_class_operator_runbook.html" in world_class_action["source_fix"], world_class_action
assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4, full_payload["data"]["world_class_evidence_ledger"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_submission_review"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_submission_review"]
assert full_payload["data"]["world_class_submission_review"]["summary"]["review_counts_submission_as_completion"] is False, full_payload["data"]["world_class_submission_review"]
assert full_payload["data"]["world_class_submission_review"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_submission_review"]
assert full_payload["data"]["world_class_operator_runbook"]["summary"]["decision"] == "collect-evidence", full_payload["data"]["world_class_operator_runbook"]
assert full_payload["data"]["world_class_operator_runbook"]["summary"]["runbook_counts_as_completion"] is False, full_payload["data"]["world_class_operator_runbook"]
assert full_payload["data"]["world_class_operator_runbook"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_operator_runbook"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["template_pass_count"] == 4, full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_count"] == 4, full_payload["data"]["world_class_evidence_intake"]
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_ready_count"] == 0, full_payload["data"]["world_class_evidence_intake"]
@@ -515,6 +524,7 @@ def main() -> None:
assert "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" in html, html
assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html
assert "reports/world_class_evidence_intake.md" in html, html
assert "reports/world_class_operator_runbook.html" in html, html
assert "reports/world_class_claim_guard.md" in html, html
assert "英文完成断言、true 状态声明或中文完成态" in html, html
assert "world-evidence-grid" in html, html
tests/verify_world_class_operator_runbook.py
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_world_class_operator_runbook.py"
CLI = ROOT / "scripts" / "yao.py"
TMP = ROOT / "tests" / "tmp_world_class_operator_runbook"
def provider_submission() -> dict:
return {
"schema_version": "1.0",
"evidence_key": "provider-holdout",
"template_only": False,
"category": "external",
"source_type": "provider-output-eval",
"submitted_by": "Yao provider operator",
"submitted_at": "2026-06-14",
"summary": "Aggregate provider-backed holdout evidence for ledger review.",
"artifact_refs": [
{
"path": "reports/output_execution_runs.json",
"kind": "aggregate-report",
"contains_raw_content": False,
"sha256": "example-only",
}
],
"provenance": {"provider": "openai", "model": "gpt-4.1-mini", "credential_material_committed": False},
"privacy": {
"raw_user_content_included": False,
"raw_provider_prompt_included": False,
"credentials_included": False,
"secrets_included": False,
},
"anti_overclaim": {
"planned_work_counts_as_evidence": False,
"metadata_fallback_counts_as_native_enforcement": False,
"pending_review_counts_as_human_decision": False,
"local_command_runner_counts_as_provider_model": False,
},
"attestation": {
"real_external_or_human_evidence": True,
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
},
}
def run_direct(*extra: str) -> dict:
proc = subprocess.run(
[sys.executable, str(SCRIPT), str(ROOT), "--generated-at", "2026-06-14", *extra],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
return json.loads(proc.stdout)
def run_cli(*extra: str) -> dict:
env = dict(os.environ)
env["YAO_CLI_TELEMETRY"] = "0"
env.pop("YAO_CLI_TELEMETRY_EVENTS", None)
proc = subprocess.run(
[sys.executable, str(CLI), *extra],
cwd=ROOT,
capture_output=True,
text=True,
env=env,
check=True,
)
return json.loads(proc.stdout)
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
TMP.mkdir(parents=True, exist_ok=True)
output_json = TMP / "world_class_operator_runbook.json"
output_md = TMP / "world_class_operator_runbook.md"
output_html = TMP / "world_class_operator_runbook.html"
payload = run_direct("--output-json", str(output_json), "--output-md", str(output_md), "--output-html", str(output_html))
assert payload["schema_version"] == "1.0", payload
assert payload["ok"] is True, payload
summary = payload["summary"]
assert summary["decision"] == "collect-evidence", summary
assert summary["evidence_item_count"] == 4, summary
assert summary["pending_count"] == 4, summary
assert summary["awaiting_submission_count"] == 4, summary
assert summary["ready_for_ledger_review_count"] == 0, summary
assert summary["valid_packet_source_incomplete_count"] == 0, summary
assert summary["invalid_submission_count"] == 0, summary
assert summary["ready_to_claim_world_class"] is False, summary
assert summary["runbook_counts_as_completion"] is False, summary
items = {item["evidence_key"]: item for item in payload["items"]}
assert set(items) == {
"provider-holdout",
"human-adjudication",
"native-permission-enforcement",
"native-client-telemetry",
}, items
provider = items["provider-holdout"]
assert provider["review_state"] == "awaiting-submission", provider
assert provider["source_accepted"] is False, provider
assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider
assert "world-class-intake" in provider["commands"]["validate_intake"], provider
assert "world-class-submission-review" in provider["commands"]["review_queue"], provider
assert "world-class-ledger" in provider["commands"]["refresh_ledger"], provider
assert "world-class-claim-guard" in provider["commands"]["guard_claim"], provider
assert "provider-backed model run" in provider["must_collect"]["provenance_requirements"], provider
assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider["must_collect"]["success_checks"], provider
markdown = output_md.read_text(encoding="utf-8")
assert "World-Class Operator Runbook" in markdown, markdown
assert "runbook counts as completion: `false`" in markdown, markdown
assert "Valid intake means ready for ledger review, not accepted evidence." in markdown, markdown
html = output_html.read_text(encoding="utf-8")
assert "World-Class Operator Runbook" in html, html[:400]
assert "ledger and claim guard" in html, html
assert "position:sticky" in html, html
assert "<script" not in html.lower(), html
assert "http://" not in html and "https://" not in html, html
cli_payload = run_cli(
"world-class-runbook",
str(ROOT),
"--output-json", str(TMP / "cli_runbook.json"),
"--output-md", str(TMP / "cli_runbook.md"),
"--output-html", str(TMP / "cli_runbook.html"),
"--generated-at", "2026-06-14",
)
assert cli_payload["summary"]["decision"] == "collect-evidence", cli_payload
assert cli_payload["artifacts"]["html"].endswith("tests/tmp_world_class_operator_runbook/cli_runbook.html"), cli_payload
submissions = TMP / "valid_submissions"
submissions.mkdir()
(submissions / "provider-holdout.json").write_text(
json.dumps(provider_submission(), ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
submitted = run_direct(
"--submissions-dir", str(submissions),
"--output-json", str(TMP / "submitted_runbook.json"),
"--output-md", str(TMP / "submitted_runbook.md"),
"--output-html", str(TMP / "submitted_runbook.html"),
)
submitted_summary = submitted["summary"]
assert submitted_summary["awaiting_submission_count"] == 3, submitted_summary
assert submitted_summary["valid_packet_source_incomplete_count"] == 1, submitted_summary
assert submitted_summary["ready_for_ledger_review_count"] == 0, submitted_summary
assert submitted_summary["ready_to_claim_world_class"] is False, submitted_summary
submitted_provider = {item["evidence_key"]: item for item in submitted["items"]}["provider-holdout"]
assert submitted_provider["intake_readiness"] == "ready-for-ledger-review", submitted_provider
assert submitted_provider["review_state"] == "source-evidence-incomplete", submitted_provider
assert submitted_provider["source_accepted"] is False, submitted_provider
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
tests/verify_yao_cli.py
+7 -18
View File
@@ -76,24 +76,13 @@ def main() -> None:
assert yao_cli_report_commands.SCRIPT_INTERFACE == "internal-module"
assert callable(yao_cli_module.command_review_studio)
parser_help = yao_cli_module.build_parser().format_help()
assert "quickstart" in parser_help, parser_help
assert "review-studio" in parser_help, parser_help
assert "python-compat" in parser_help, parser_help
assert "architecture-audit" in parser_help, parser_help
assert "skill-os2-audit" in parser_help, parser_help
assert "skill-os2-coverage" in parser_help, parser_help
assert "world-class-evidence" in parser_help, parser_help
assert "world-class-ledger" in parser_help, parser_help
assert "world-class-intake" in parser_help, parser_help
assert "world-class-submission-kit" in parser_help, parser_help
assert "world-class-submission-review" in parser_help, parser_help
assert "world-class-claim-guard" in parser_help, parser_help
assert "benchmark-reproducibility" in parser_help, parser_help
assert "output-review-kit" in parser_help, parser_help
assert "telemetry-import" in parser_help, parser_help
assert "telemetry-emit" in parser_help, parser_help
assert "telemetry-hooks" in parser_help, parser_help
assert "--record-cli-telemetry" in parser_help, parser_help
expected_help = (
"quickstart review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage "
"world-class-evidence world-class-ledger world-class-intake world-class-submission-kit "
"world-class-submission-review world-class-runbook world-class-claim-guard "
"benchmark-reproducibility output-review-kit telemetry-import telemetry-emit telemetry-hooks --record-cli-telemetry"
).split()
assert all(item in parser_help for item in expected_help), parser_help
init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
assert init_result["ok"], init_result