diff --git a/AGENTS.md b/AGENTS.md index 586f8b10..04eddb10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT" python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT" python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT" python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json @@ -75,6 +76,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. - `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence. - `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state. +- `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness. - `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. diff --git a/Makefile b/Makefile index 449e2c91..f92f1469 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -112,6 +112,9 @@ world-class-intake-check: world-class-submission-review-check: $(PYTHON) tests/verify_world_class_submission_review.py +world-class-runbook-check: + $(PYTHON) tests/verify_world_class_operator_runbook.py + world-class-claim-guard-check: $(PYTHON) tests/verify_world_class_claim_guard.py @@ -232,11 +235,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/registry/index.json b/registry/index.json index 4057c3cd..b9c0d751 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,6 +1,6 @@ { "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "package_count": 1, "packages": [ { @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index d3f060bc..7837ed7b 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771", - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, "compatibility": { "openai": "pass", @@ -48,10 +48,10 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-13" + "generated_at": "2026-06-14" } diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 0eb11482..8ed61f08 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T23:43:44Z", + "generated_at": "2026-06-14T00:05:43Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 59011737..b188c644 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,11 +4,11 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 151, - "script_file_count": 93, - "test_file_count": 58, + "python_file_count": 153, + "script_file_count": 94, + "test_file_count": 59, "internal_module_count": 19, - "cli_script_count": 75, + "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, "block_line_threshold": 1500, @@ -27,14 +27,14 @@ }, { "path": "tests/verify_yao_cli.py", - "lines": 899, + "lines": 888, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 867, + "lines": 874, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -47,16 +47,16 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/compile_skill.py", - "lines": 734, - "kind": "cli-script", + "path": "scripts/yao_cli_parser.py", + "lines": 741, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/yao_cli_parser.py", - "lines": 729, - "kind": "internal-module", + "path": "scripts/compile_skill.py", + "lines": 734, + "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index a50a3855..168317fb 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,11 +5,11 @@ Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `151` -- scripts: `93` -- tests: `58` +- python files: `153` +- scripts: `94` +- tests: `59` - internal modules: `19` -- CLI scripts: `75` +- CLI scripts: `76` - Yao CLI command handlers: `34` - largest file lines: `899` - hotspots: `0` @@ -26,11 +26,11 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `scripts/render_review_studio.py` | `899` | `cli-script` | `pass` | -| `tests/verify_yao_cli.py` | `899` | `test` | `pass` | -| `scripts/yao.py` | `867` | `cli-script` | `pass` | +| `tests/verify_yao_cli.py` | `888` | `test` | `pass` | +| `scripts/yao.py` | `874` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `792` | `internal-module` | `pass` | +| `scripts/yao_cli_parser.py` | `741` | `internal-module` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | -| `scripts/yao_cli_parser.py` | `729` | `internal-module` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 7d8bdb74..6d0537dd 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,24 +3,24 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130", + "commit": "08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1", "git_status": { "available": true, "dirty": true, - "changed_file_count": 49, + "changed_file_count": 61, "sample": [ - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", - " M reports/adoption_drift_report.json", + "M AGENTS.md", + "M Makefile", + "M registry/index.json", + "M registry/packages/yao-meta-skill.json", + "MM reports/adoption_drift_report.json", " M reports/adoption_drift_report.md", - " M reports/architecture_maintainability.json", - " M reports/architecture_maintainability.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", - " M reports/compiled_targets.json", - " M reports/context_budget.json", - " M reports/install_simulation.json", - " M reports/install_simulation.md" + "MM reports/architecture_maintainability.json", + "MM reports/architecture_maintainability.md", + "MM reports/benchmark_reproducibility.json", + "MM reports/benchmark_reproducibility.md", + "MM reports/compiled_targets.json", + "MM reports/context_budget.json" ] }, "summary": { @@ -42,7 +42,7 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 49 + "changed_file_count": 61 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -115,8 +115,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "b0953427365ce84f89454531d5819a2ce6f494875661fbbb3be33b012a0d8f13" + "bytes": 7965, + "sha256": "99b2e3f03710fdcee8fe66a476bc589f4e22a379bfcc1cd5561de49c184286e8" }, { "label": "blind_review", @@ -150,43 +150,43 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 95873, - "sha256": "cb599bf68524be22e7df189f1f09deac404bda493a3013c74be209d7ef7efae4" + "bytes": 97005, + "sha256": "252c580028df6ac2404b53cc10bb607fa82b98ef17965cb7354c4731f6c6b08e" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 20093, - "sha256": "e5dd973d0b5c78c5e40d52727939134b8167d742cb90bea2141744c0f7d5fa33" + "bytes": 20367, + "sha256": "a8d14087863459534fadac760a64c213c9474413e92046a9ac2e772d4174d347" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "2ff87aeb6f3dc7f0614d33d291faad1a55da2ade3ba98014bee5143dfb8f4806" + "sha256": "6527b723a4bc367477e1090afbf7f1653f20d33fed0e0691725b7ee30896848b" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "a5dac0ea2b6365c97d0b72ad695ed0ebbd5ac472745e20c1a98f695b26bdd827" + "sha256": "6bf19c9e29803ad8668914489d22b9e306ad0ad092c91c179c3aaffe81eed2c6" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "eebfba42d71e4082d3c801fd126e59c6401b3f8c00a00625c70eb82f927ebd65" + "bytes": 8758, + "sha256": "270a28b82dc49d80ba9c5a82f912cc960e50a5ac42f022c6a2c345e4e84fbb36" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14309, - "sha256": "78bcbcbc54711e63a6e34059c30748599a7e71d434182b78c307af6d2553cc81" + "sha256": "cad5296cd500a40cb0d83897d327b3542f31c1ce6bfe7eaa387ea2e88d229aa1" }, { "label": "world_class_evidence_plan", @@ -213,8 +213,8 @@ "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, - "bytes": 8267, - "sha256": "79c5b29f0ff639189aa65f3487afab13150e95c7fa7a6a5b59309b1e87873c25" + "bytes": 8463, + "sha256": "250d616b028cf046e2033f9e2c5648c8d95c110d0d8990b35e4d481e14cfc558" } ], "missing_artifacts": [], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 2b0c6751..0d4fa698 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130` +Commit: `08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1` Working tree dirty at generation: `true` ## Summary @@ -16,7 +16,7 @@ Working tree dirty at generation: `true` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `49` +- changed files at generation: `61` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `b0953427365c` | +| output_execution | `reports/output_execution_runs.json` | present | `99b2e3f03710` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `cb599bf68524` | -| python_compatibility | `reports/python_compatibility.json` | present | `e5dd973d0b5c` | -| registry_audit | `reports/registry_audit.json` | present | `2ff87aeb6f3d` | -| package_verification | `reports/package_verification.json` | present | `a5dac0ea2b63` | -| install_simulation | `reports/install_simulation.json` | present | `eebfba42d71e` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `78bcbcbc5471` | +| trust_report | `reports/security_trust_report.json` | present | `252c580028df` | +| python_compatibility | `reports/python_compatibility.json` | present | `a8d140878634` | +| registry_audit | `reports/registry_audit.json` | present | `6527b723a4bc` | +| package_verification | `reports/package_verification.json` | present | `6bf19c9e2980` | +| install_simulation | `reports/install_simulation.json` | present | `270a28b82dc4` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `cad5296cd500` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `02e49b66159d` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0b52d0d1528b` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `5fbfcd35ac6a` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `79c5b29f0ff6` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` | ## Reproduction Commands diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index 77a50459..dfb56a41 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "target_count": 5, @@ -205,6 +205,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -223,8 +224,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -383,7 +383,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -437,6 +437,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -486,14 +487,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -515,7 +516,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -628,7 +629,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -682,6 +683,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -731,14 +733,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -758,7 +760,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -1052,6 +1054,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -1070,8 +1073,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -1230,7 +1232,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1284,6 +1286,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -1333,14 +1336,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1362,7 +1365,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -1475,7 +1478,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1529,6 +1532,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -1578,14 +1582,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1605,7 +1609,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -1899,6 +1903,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -1917,8 +1922,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2077,7 +2081,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2131,6 +2135,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -2180,14 +2185,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2209,7 +2214,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -2315,7 +2320,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2369,6 +2374,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -2418,14 +2424,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2445,7 +2451,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -2730,6 +2736,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -2748,8 +2755,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2908,7 +2914,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2962,6 +2968,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3011,14 +3018,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3040,7 +3047,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -3146,7 +3153,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3200,6 +3207,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3249,14 +3257,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3276,7 +3284,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -3561,6 +3569,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -3579,8 +3588,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -3739,7 +3747,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3793,6 +3801,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3842,14 +3851,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3871,7 +3880,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -3981,7 +3990,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4035,6 +4044,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -4084,14 +4094,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4111,7 +4121,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, diff --git a/reports/context_budget.json b/reports/context_budget.json index 0bd1cf6c..e6228caf 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1157142, + "other_text_tokens": 1175718, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1157893, - "relevant_file_count": 491, + "estimated_total_text_tokens": 1176469, + "relevant_file_count": 498, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index d9c66f28..291d2ad4 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -1,14 +1,14 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", - "package_dir": "tests/tmp_review_studio/dist", - "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", - "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "dist", + "install_root": "dist/install-simulation/simulate-yao-meta-skill", + "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 568, + "archive_entry_count": 573, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -26,7 +26,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" + "detail": "Package archive exists: dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -272,9 +272,9 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", - "package_manifest": "tests/tmp_review_studio/dist/manifest.json", - "json": "tests/tmp_review_studio/install_simulation.json", - "markdown": "tests/tmp_review_studio/install_simulation.md" + "archive": "dist/yao-meta-skill.zip", + "package_manifest": "dist/manifest.json", + "json": "reports/install_simulation.json", + "markdown": "reports/install_simulation.md" } } diff --git a/reports/install_simulation.md b/reports/install_simulation.md index b0d16dae..6750e007 100644 --- a/reports/install_simulation.md +++ b/reports/install_simulation.md @@ -1,7 +1,7 @@ # Install Simulation - OK: `True` -- Package directory: `tests/tmp_review_studio/dist` +- Package directory: `dist` - Archive extracted: `True` - Entrypoint loaded: `True` - Manifest loaded: `True` @@ -16,7 +16,7 @@ | Check | Status | Detail | | --- | --- | --- | -| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip | +| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | | `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 1a1da6df..8a2522ac 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.97, + "duration_ms": 26.47, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.97, + "duration_ms": 25.78, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.26, + "duration_ms": 25.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.71, + "duration_ms": 29.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.89, + "duration_ms": 29.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.93, + "duration_ms": 28.81, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.5, + "duration_ms": 25.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.37, + "duration_ms": 26.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.2, + "duration_ms": 25.43, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.27, + "duration_ms": 25.88, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index f5e09b95..57ab3e4f 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.97 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 27.97 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 28.26 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.71 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.89 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.93 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.5 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.37 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.2 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.27 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.47 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 25.78 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 25.73 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.11 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.2 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.81 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.65 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.22 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.43 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.88 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 29dc62b0..9e8105cd 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", - "archive_entry_count": 568, + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", + "archive_entry_count": 573, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index e0ce4671..0013353c 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b` +- Archive SHA256: `de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 290a9303..850f303b 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 154, + "file_count": 156, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -400,6 +400,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/render_world_class_submission_review.py", "ok": true, @@ -934,6 +940,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_world_class_submission_review.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index d4e290ef..c9259ec3 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-13` - decision: `pass` - target python: `3.11` -- files scanned: `154` +- files scanned: `156` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 341f44a7..ee90b8ea 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771", - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, "compatibility": { "openai": "pass", @@ -53,16 +53,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-13" + "generated_at": "2026-06-14" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "package_count": 1, "packages": [ { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 85cdccfe..040a4b4e 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771` -- Archive SHA256: `cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b` +- Package SHA256: `97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179` +- Archive SHA256: `de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index fd49d8fb..5b809fbc 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -434,12 +434,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

93 scripts scanned; secrets found

Py Compat0

154 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

71 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

568 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

94 scripts scanned; secrets found

Py Compat0

156 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

573 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

151 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -449,7 +449,7 @@

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
@@ -465,7 +465,7 @@

证据路径

- +
@@ -506,12 +506,12 @@
-

信任报告

Secret
0
脚本数
93
网络脚本
3
Help 失败
0
包体哈希
7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771
+

信任报告

Secret
0
脚本数
94
网络脚本
3
Help 失败
0
包体哈希
97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
154
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
156
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线;本地更高版本允许的新语法不能绕过兼容门禁。

@@ -568,18 +568,18 @@
-

声明守卫

台账可声明
台账待补
4
声明面
71
违规数
0
Overclaim Guard Active
+

声明守卫

台账可声明
台账待补
4
声明面
73
违规数
0
Overclaim Guard Active

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述;ledger 未 ready 时,任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
568
失败数
0
警告数
0
归档哈希
cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
573
失败数
0
警告数
0
归档哈希
de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e
diff --git a/reports/review-studio.json b/reports/review-studio.json index f4e4fed5..1e66ced7 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "151 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers", + "detail": "153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -271,7 +271,7 @@ "priority": "warning", "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。", "why": "世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。", - "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", + "source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", "source_refs": [ { "path": "reports/world_class_evidence_ledger.md", @@ -372,7 +372,7 @@ ], "evidence": "reports/world_class_evidence_ledger.json", "evidence_link": "world_class_evidence_ledger.md", - "verification_command": "python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio ." + "verification_command": "python3 scripts/yao.py world-class-runbook . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio ." } ], "evidence_paths": { @@ -402,6 +402,8 @@ "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", "world_class_submission_review": "reports/world_class_submission_review.md", + "world_class_operator_runbook": "reports/world_class_operator_runbook.md", + "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", "world_class_claim_guard": "reports/world_class_claim_guard.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", @@ -849,7 +851,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 93 + "file_count": 94 }, { "path": "evals", @@ -861,10 +863,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 200 + "file_count": 203 } ], - "file_count": 357, + "file_count": 361, "folder_count": 4, "distribution": [ { @@ -889,7 +891,7 @@ }, { "label": "scripts", - "value": 93 + "value": 94 }, { "label": "evals", @@ -897,7 +899,7 @@ }, { "label": "reports", - "value": 200 + "value": 203 } ] }, @@ -1018,7 +1020,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 93 + "file_count": 94 }, { "path": "evals", @@ -1030,7 +1032,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 200 + "file_count": 203 } ], "strengths": [ @@ -1347,9 +1349,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 32 + "changed_file_count": 61 }, - "commit": "6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130", + "commit": "08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1432,8 +1434,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 177, - "script_count": 93, + "scanned_files": 178, + "script_count": 94, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -1442,18 +1444,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 63, + "file_write_script_count": 64, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 76, + "help_smoke_checked_count": 77, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 177, - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_hash_file_count": 178, + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" }, "skill_atlas": { "skill_count": 12, @@ -1491,8 +1493,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771", - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, "compatibility": { "openai": "pass", @@ -1523,7 +1525,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1539,8 +1541,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", - "archive_entry_count": 568, + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", + "archive_entry_count": 573, "failure_count": 0, "warning_count": 0 }, @@ -1551,7 +1553,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 568, + "archive_entry_count": 573, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1618,12 +1620,12 @@ { "field": "archive_sha256", "from": "", - "to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, @@ -3792,7 +3794,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.32, + "duration_ms": 26.47, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3820,7 +3822,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.41, + "duration_ms": 25.78, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3843,7 +3845,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.48, + "duration_ms": 25.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3871,7 +3873,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.35, + "duration_ms": 29.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3894,7 +3896,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.32, + "duration_ms": 29.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3922,7 +3924,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.82, + "duration_ms": 28.81, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3945,7 +3947,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.54, + "duration_ms": 25.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3972,7 +3974,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.39, + "duration_ms": 26.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3995,7 +3997,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.46, + "duration_ms": 25.43, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4024,7 +4026,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.36, + "duration_ms": 25.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4864,7 +4866,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "93 scripts; secrets 0; help failures 0", + "current": "94 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -4938,7 +4940,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 568; install failures 0", + "current": "archive entries 573; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -5563,6 +5565,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -5581,8 +5584,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -5741,7 +5743,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5795,6 +5797,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -5844,14 +5847,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5873,7 +5876,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -5986,7 +5989,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6040,6 +6043,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -6089,14 +6093,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6116,7 +6120,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -6410,6 +6414,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -6428,8 +6433,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -6588,7 +6592,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6642,6 +6646,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -6691,14 +6696,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6720,7 +6725,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -6833,7 +6838,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6887,6 +6892,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -6936,14 +6942,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6963,7 +6969,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -7257,6 +7263,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -7275,8 +7282,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -7435,7 +7441,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7489,6 +7495,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -7538,14 +7545,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7567,7 +7574,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -7673,7 +7680,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7727,6 +7734,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -7776,14 +7784,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7803,7 +7811,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -8088,6 +8096,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -8106,8 +8115,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -8266,7 +8274,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8320,6 +8328,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -8369,14 +8378,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8398,7 +8407,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -8504,7 +8513,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8558,6 +8567,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -8607,14 +8617,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8634,7 +8644,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -8919,6 +8929,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -8937,8 +8948,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -9097,7 +9107,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -9151,6 +9161,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -9200,14 +9211,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -9229,7 +9240,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -9339,7 +9350,7 @@ }, "file_write": { "required": true, - "script_count": 63, + "script_count": 64, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -9393,6 +9404,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -9442,14 +9454,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 76, + "checked_count": 77, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 63, + "file_write_script_count": 64, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -9469,7 +9481,7 @@ ], "capability_counts": { "network": 3, - "file_write": 63, + "file_write": 64, "subprocess": 9, "interactive": 0 }, @@ -9818,7 +9830,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -9858,7 +9870,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -9955,7 +9967,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10037,7 +10049,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10119,7 +10131,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -10209,8 +10221,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 177, - "script_count": 93, + "scanned_files": 178, + "script_count": 94, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -10219,18 +10231,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 63, + "file_write_script_count": 64, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 76, + "help_smoke_checked_count": 77, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 177, - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_hash_file_count": 178, + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" }, "failures": [], "warnings": [], @@ -11093,6 +11105,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders an operator runbook for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_world_class_submission_review.py", "interface": "cli", @@ -11586,9 +11612,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 76, - "checked_count": 76, - "passed_count": 76, + "candidate_count": 77, + "checked_count": 77, + "passed_count": 77, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -12193,6 +12219,16 @@ "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "command": "python3 scripts/render_world_class_operator_runbook.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_world_class_submission_review.py", "command": "python3 scripts/render_world_class_submission_review.py --help", @@ -12529,6 +12565,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -12607,7 +12644,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 154, + "file_count": 156, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -13002,6 +13039,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/render_world_class_submission_review.py", "ok": true, @@ -13536,6 +13579,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_world_class_submission_review.py", "ok": true, @@ -13561,11 +13610,11 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 151, - "script_file_count": 93, - "test_file_count": 58, + "python_file_count": 153, + "script_file_count": 94, + "test_file_count": 59, "internal_module_count": 19, - "cli_script_count": 75, + "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, "block_line_threshold": 1500, @@ -13584,14 +13633,14 @@ }, { "path": "tests/verify_yao_cli.py", - "lines": 899, + "lines": 888, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 867, + "lines": 874, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -13604,16 +13653,16 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/compile_skill.py", - "lines": 734, - "kind": "cli-script", + "path": "scripts/yao_cli_parser.py", + "lines": 741, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/yao_cli_parser.py", - "lines": 729, - "kind": "internal-module", + "path": "scripts/compile_skill.py", + "lines": 734, + "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, @@ -13675,10 +13724,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1157129, + "other_text_tokens": 1175718, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1157880, - "relevant_file_count": 491, + "estimated_total_text_tokens": 1176469, + "relevant_file_count": 498, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -14602,6 +14651,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -15530,7 +15580,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T23:42:36Z", + "generated_at": "2026-06-14T00:05:43Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -16433,6 +16483,325 @@ "markdown": "reports/world_class_submission_review.md" } }, + "world_class_operator_runbook": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-14", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "pending_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "ready_for_ledger_review_count": 0, + "valid_packet_source_incomplete_count": 0, + "invalid_submission_count": 0, + "ready_to_claim_world_class": false, + "runbook_counts_as_completion": false, + "decision": "collect-evidence" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "runbook_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "current": "model-executed 0; token-observed 0", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "current": "0/5 decisions; pending 5", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "current": "native-enforced targets 0; installer-enforced targets 4", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "current": "external source events 0; adoption samples 1", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 1, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + } + ], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json" + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html" + } + }, "world_class_claim_guard": { "schema_version": "1.0", "ok": true, @@ -16441,7 +16810,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 71, + "claim_surface_count": 73, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -16750,6 +17119,14 @@ "path": "reports/world_class_evidence_plan.md", "violation_count": 0 }, + { + "path": "reports/world_class_operator_runbook.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.md", + "violation_count": 0 + }, { "path": "reports/world_class_submission_review.md", "violation_count": 0 @@ -16792,8 +17169,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771", - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, "compatibility": { "openai": "pass", @@ -16824,7 +17201,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -16849,7 +17226,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, @@ -16872,8 +17249,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", - "archive_entry_count": 568, + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", + "archive_entry_count": 573, "failure_count": 0, "warning_count": 0 }, @@ -17538,7 +17915,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 568, + "archive_entry_count": 573, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -17880,12 +18257,12 @@ { "field": "archive_sha256", "from": "", - "to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 3de9b458..c6f4bbd8 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 177, - "script_count": 93, + "scanned_files": 178, + "script_count": 94, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 63, + "file_write_script_count": 64, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 76, + "help_smoke_checked_count": 77, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 177, - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_hash_file_count": 178, + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" }, "failures": [], "warnings": [], @@ -886,6 +886,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders an operator runbook for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_world_class_submission_review.py", "interface": "cli", @@ -1379,9 +1393,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 76, - "checked_count": 76, - "passed_count": 76, + "candidate_count": 77, + "checked_count": 77, + "passed_count": 77, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -1986,6 +2000,16 @@ "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "command": "python3 scripts/render_world_class_operator_runbook.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_world_class_submission_review.py", "command": "python3 scripts/render_world_class_submission_review.py --help", @@ -2322,6 +2346,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 02c2cbbd..71e1e1dc 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `177` -- Scripts: `93` +- Scanned files: `178` +- Scripts: `94` - Internal script modules: `17` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `63` +- File-write scripts: `64` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `76` +- CLI help smoke checked: `77` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `177` -- Package SHA256: `7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771` +- Package hash files: `178` +- Package SHA256: `97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `76` -- Passed scripts: `76` +- Checked scripts: `77` +- Passed scripts: `77` - Failed scripts: `none` ## Script Surface @@ -122,6 +122,7 @@ | scripts/render_world_class_evidence_intake.py | cli | True | True | True | False | False | True | False | Validates world-class human and external evidence intake packets before ledger review. | | scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. | | scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_world_class_operator_runbook.py | cli | True | True | True | False | False | True | False | Renders an operator runbook for collecting pending world-class evidence without accepting evidence. | | scripts/render_world_class_submission_review.py | cli | True | True | True | False | False | True | False | Renders a read-only review queue for world-class evidence submissions before ledger acceptance. | | scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/review_studio_data.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to load Review Studio source reports and metric cards. | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 6e6c072b..edee53be 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布357项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布361项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index ce02707e..dc51038d 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -437,7 +437,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 93 + "file_count": 94 }, { "path": "evals", @@ -449,10 +449,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 200 + "file_count": 203 } ], - "file_count": 357, + "file_count": 361, "folder_count": 4, "distribution": [ { @@ -477,7 +477,7 @@ }, { "label": "scripts", - "value": 93 + "value": 94 }, { "label": "evals", @@ -485,7 +485,7 @@ }, { "label": "reports", - "value": 200 + "value": 203 } ] }, @@ -606,7 +606,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 93 + "file_count": 94 }, { "path": "evals", @@ -618,7 +618,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 200 + "file_count": 203 } ], "strengths": [ @@ -935,9 +935,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 32 + "changed_file_count": 61 }, - "commit": "6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130", + "commit": "08347d5c2cae9528ae1fe3c50ad39bb3c773a6e1", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1020,8 +1020,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 177, - "script_count": 93, + "scanned_files": 178, + "script_count": 94, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -1030,18 +1030,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 63, + "file_write_script_count": 64, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 76, + "help_smoke_checked_count": 77, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 177, - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "package_hash_file_count": 178, + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" }, "skill_atlas": { "skill_count": 12, @@ -1079,8 +1079,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771", - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "package_sha256": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, "compatibility": { "openai": "pass", @@ -1111,7 +1111,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1127,8 +1127,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b", - "archive_entry_count": 568, + "archive_sha256": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e", + "archive_entry_count": 573, "failure_count": 0, "warning_count": 0 }, @@ -1139,7 +1139,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 568, + "archive_entry_count": 573, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1206,12 +1206,12 @@ { "field": "archive_sha256", "from": "", - "to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index eaef5d12..09f7c547 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -124,6 +124,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 7b0a8f44..e7d4abd6 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 93; help failures 0", + "current": "secrets 0; scripts 94; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 568; install failures 0; permission failures 0", + "current": "zip entries 573; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 414db533..ab20a7eb 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-13` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 20; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 93; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 94; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 568; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 573; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index e4766103..654e1583 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -122,7 +122,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "93 scripts; secrets 0; help failures 0", + "current": "94 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -196,7 +196,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 568; install failures 0", + "current": "archive entries 573; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index d2a8c5dd..33af7009 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 93 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 94 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 568; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 573; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 88dfaa30..43cf1107 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "cc5a43f9a1a3cf2a75cd3dc31d1317b90b2a911f32e3667b62936943dc5d808b" + "to": "de2056721d0fddfbae7fbde46493213c0a5a2879ed9dd6b3875ae4625ae2072e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7f70f5ec6944b938933be61ba59499f6f65493c99a72213c59609b0f0cbd4771" + "to": "97ae1c187b764f2504ead4bf0c3063b466f8e8daef0de0ed8001d2722ea2d179" } ] }, @@ -90,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "tests/tmp_review_studio/upgrade_check.json", - "markdown": "tests/tmp_review_studio/upgrade_check.md" + "json": "reports/upgrade_check.json", + "markdown": "reports/upgrade_check.md" } } diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index f02c3351..88dc21e5 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -1,12 +1,12 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 71, + "claim_surface_count": 73, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -315,6 +315,14 @@ "path": "reports/world_class_evidence_plan.md", "violation_count": 0 }, + { + "path": "reports/world_class_operator_runbook.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.md", + "violation_count": 0 + }, { "path": "reports/world_class_submission_review.md", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index cc998c5a..b3eaa58f 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -1,13 +1,13 @@ # World-Class Claim Guard -Generated at: `2026-06-13` +Generated at: `2026-06-14` ## Summary - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `71` +- claim surfaces scanned: `73` - violations: `0` - overclaim guard active: `true` diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index c28c3345..633de5e4 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "schema_present": true, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index 0ac63b34..d14b0939 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -1,6 +1,6 @@ # World-Class Evidence Intake -Generated at: `2026-06-13` +Generated at: `2026-06-14` ## Summary diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index 7b2c05a6..efea089e 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "ledger_entry_count": 4, diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 281e29ca..e6b0f882 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -1,6 +1,6 @@ # World-Class Evidence Ledger -Generated at: `2026-06-13` +Generated at: `2026-06-14` ## Summary diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index afc0abe8..ae80126b 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "audit_decision": "continue-iteration", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 7f5d2c16..c8f1d162 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -1,6 +1,6 @@ # World-Class Evidence Plan -Generated at: `2026-06-13` +Generated at: `2026-06-14` ## Summary diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html new file mode 100644 index 00000000..a7533147 --- /dev/null +++ b/reports/world_class_operator_runbook.html @@ -0,0 +1,118 @@ + + + + + + World-Class Operator Runbook + + + + +
+
+ Evidence Operations +

World-Class Operator Runbook

+

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

+
Pending4
Awaiting4
Ready
Invalid
+
+

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.
+

Evidence Items

+
external · awaiting-submission

Provider Holdout

+

No evidence packet has been submitted for review.

+
+
Owner
operator with provider credentials
+
Ledger
pending
+
Submission
evidence/world_class/submissions/provider-holdout.json
+
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger .
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • provider-backed model run
  • observed timing
  • observed token metadata
+

Success Checks

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
+

Privacy

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
+
+
+
human · awaiting-submission

Human Adjudication

+

No evidence packet has been submitted for review.

+
+
Owner
human reviewer
+
Ledger
pending
+
Submission
evidence/world_class/submissions/human-adjudication.json
+
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger .
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
+

Success Checks

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
+

Privacy

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
+
+
+
external · awaiting-submission

Native Permission Enforcement

+

No evidence packet has been submitted for review.

+
+
Owner
target client or installer integrator
+
Ledger
pending
+
Submission
evidence/world_class/submissions/native-permission-enforcement.json
+
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger .
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
+

Success Checks

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
+

Privacy

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
+
+
+
external · awaiting-submission

Native Client Telemetry

+

No evidence packet has been submitted for review.

+
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Ledger
pending
+
Submission
evidence/world_class/submissions/native-client-telemetry.json
+
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger .
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real external client source
  • metadata-only event
  • local-first import path
+

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
+

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
+
+

Boundary

  • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
  • Valid intake means ready for ledger review, not accepted evidence.
  • The world-class ledger and claim guard remain the source of truth.
+
+ + diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json new file mode 100644 index 00000000..c7f73901 --- /dev/null +++ b/reports/world_class_operator_runbook.json @@ -0,0 +1,319 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-14", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "pending_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "ready_for_ledger_review_count": 0, + "valid_packet_source_incomplete_count": 0, + "invalid_submission_count": 0, + "ready_to_claim_world_class": false, + "runbook_counts_as_completion": false, + "decision": "collect-evidence" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "runbook_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "current": "model-executed 0; token-observed 0", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "current": "0/5 decisions; pending 5", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "current": "native-enforced targets 0; installer-enforced targets 4", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "current": "external source events 0; adoption samples 1", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 1, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + } + ], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json" + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html" + } +} diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md new file mode 100644 index 00000000..449a0dc2 --- /dev/null +++ b/reports/world_class_operator_runbook.md @@ -0,0 +1,214 @@ +# World-Class Operator Runbook + +Generated at: `2026-06-14` + +## Summary + +- decision: `collect-evidence` +- ready to claim world-class: `false` +- runbook counts as completion: `false` +- evidence items: `4` +- pending: `4` +- awaiting submission: `4` +- ready for ledger review: `0` + +This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true. + +## Fast Path + +1. Run the real external or human work for one evidence item. +2. Generate the matching submission draft. +3. Replace template-only fields with aggregate evidence and provenance. +4. Validate intake, review the queue, refresh the ledger, then run the claim guard. + +## Evidence Items + +| Evidence | Ledger | Intake | Review | Owner | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | operator with provider credentials | +| `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | human reviewer | +| `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | target client or installer integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | Browser/Chrome/IDE/provider client integrator | + +## Provider Holdout + +- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. +- blocking reason: No evidence packet has been submitted for review. +- submission: `evidence/world_class/submissions/provider-holdout.json` +- template: `evidence/world_class/templates/provider-holdout.intake.json` + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger .` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- provider-backed model run +- observed timing +- observed token metadata + +### Success Checks + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +### Evidence Artifacts + +- reports/output_execution_runs.json +- reports/output_execution_runs.md +- reports/skill_os2_audit.json +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/provider-holdout.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +## Human Adjudication + +- objective: Record real blind A/B reviewer decisions before claiming human output review completion. +- blocking reason: No evidence packet has been submitted for review. +- submission: `evidence/world_class/submissions/human-adjudication.json` +- template: `evidence/world_class/templates/human-adjudication.intake.json` + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger .` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real reviewer identity +- blind A/B decisions +- answer key unopened until decisions exist + +### Success Checks + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Keep the answer key separate until after decisions are recorded. + +### Evidence Artifacts + +- reports/output_blind_review_pack.md +- reports/output_review_kit.md +- reports/output_review_decisions.json +- reports/output_review_adjudication.json +- reports/output_review_adjudication.md +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/human-adjudication.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +## Native Permission Enforcement + +- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities. +- blocking reason: No evidence packet has been submitted for review. +- submission: `evidence/world_class/submissions/native-permission-enforcement.json` +- template: `evidence/world_class/templates/native-permission-enforcement.intake.json` + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger .` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real target client or external installer runtime guard +- native enforcement flag or externally accepted guard proof +- residual risk retained for fallback targets + +### Success Checks + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +### Evidence Artifacts + +- dist/targets/*/adapter.json +- reports/runtime_permission_probes.json +- reports/runtime_permission_probes.md +- reports/install_simulation.json +- reports/install_simulation.md +- security/permission_policy.json +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/native-permission-enforcement.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +## Native Client Telemetry + +- objective: Import production metadata-only events from a real external client into the local drift loop. +- blocking reason: No evidence packet has been submitted for review. +- submission: `evidence/world_class/submissions/native-client-telemetry.json` +- template: `evidence/world_class/templates/native-client-telemetry.intake.json` + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger .` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real external client source +- metadata-only event +- local-first import path + +### Success Checks + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. + +### Evidence Artifacts + +- reports/adoption_drift_report.json +- reports/adoption_drift_report.md +- reports/telemetry_hook_recipes.json +- scripts/telemetry_native_host.py +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/native-client-telemetry.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +## Boundary + +- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion. +- Valid intake means ready for ledger review, not accepted evidence. +- The world-class ledger and claim guard remain the source of truth. diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index f5cd09b6..b462c0bb 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "review_item_count": 4, diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 11389553..f8f7527b 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -1,6 +1,6 @@ # World-Class Submission Review -Generated at: `2026-06-13` +Generated at: `2026-06-14` ## Summary diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 8bfacaf6..e29fb1b6 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -58,6 +58,7 @@ DEFAULT_TARGETS = [ "world-class-ledger-check", "world-class-intake-check", "world-class-submission-review-check", + "world-class-runbook-check", "world-class-claim-guard-check", "benchmark-reproducibility-check", "feedback-check", diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 12cc1a33..e214e8b4 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -263,7 +263,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { "world-class-evidence": { "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。", "why": "世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。", - "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", + "source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", "source_paths": [ {"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]}, {"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]}, @@ -278,7 +278,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, {"path": "reports/adoption_drift_report.md", "label": "adoption drift", "kind": "report", "patterns": ["# Adoption"]}, ], - "verification": "python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .", + "verification": "python3 scripts/yao.py world-class-runbook . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .", }, "registry-audit": { "summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。", diff --git a/scripts/render_world_class_operator_runbook.py b/scripts/render_world_class_operator_runbook.py new file mode 100644 index 00000000..4ea7adc2 --- /dev/null +++ b/scripts/render_world_class_operator_runbook.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +import argparse +import html +import json +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger +from render_world_class_submission_review import build_submission_review + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders an operator runbook for collecting pending world-class evidence without accepting evidence." + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def html_text(value: Any) -> str: + return html.escape(str(value or ""), quote=True) + + +def by_key(items: list[dict[str, Any]], key_name: str) -> dict[str, dict[str, Any]]: + return {str(item.get(key_name, "")): item for item in items if item.get(key_name)} + + +def output_path(path: str, skill_dir: Path) -> str: + candidate = Path(path) + if candidate.is_absolute(): + return rel_path(candidate, skill_dir) + return path + + +def build_runbook_item( + entry: dict[str, Any], + checklist: dict[str, Any], + review_item: dict[str, Any], +) -> dict[str, Any]: + commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {} + must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {} + return { + "evidence_key": entry.get("key", ""), + "label": entry.get("label", entry.get("key", "")), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "ledger_status": entry.get("status", "pending"), + "intake_readiness": checklist.get("readiness", "missing"), + "review_state": review_item.get("review_state", "missing"), + "source_accepted": review_item.get("source_accepted") is True, + "objective": entry.get("objective", ""), + "current": entry.get("current", ""), + "blocking_reason": review_item.get("blocking_reason") or checklist.get("blocking_reason") or entry.get("next_action", ""), + "submission_path": checklist.get("submission_path") or entry.get("submission_state", {}).get("path", ""), + "template_path": checklist.get("template_path", ""), + "commands": { + "prepare_submission": commands.get("prepare_submission", ""), + "validate_intake": commands.get("validate_intake", ""), + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": commands.get("refresh_ledger", "python3 scripts/yao.py world-class-ledger ."), + "guard_claim": commands.get("guard_claim", "python3 scripts/yao.py world-class-claim-guard ."), + }, + "must_collect": { + "provenance_requirements": must_collect.get("provenance_requirements", entry.get("provenance_requirements", [])), + "success_checks": must_collect.get("success_checks", entry.get("success_checks", [])), + "privacy_contract": must_collect.get("privacy_contract", entry.get("privacy_contract", [])), + }, + "evidence_artifacts": entry.get("evidence_artifacts", []), + "observed_state": entry.get("observed_state", {}), + "submission_state": entry.get("submission_state", {}), + "anti_overclaim": entry.get("anti_overclaim", {}), + } + + +def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir) + checklist_by_key = by_key(intake.get("operator_checklist", []), "evidence_key") + review_by_key = by_key(review.get("items", []), "evidence_key") + items = [ + build_runbook_item(entry, checklist_by_key.get(str(entry.get("key", "")), {}), review_by_key.get(str(entry.get("key", "")), {})) + for entry in ledger.get("entries", []) + ] + summary = ledger.get("summary", {}) + review_summary = review.get("summary", {}) + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "evidence_item_count": len(items), + "pending_count": summary.get("pending_count", 0), + "accepted_count": summary.get("accepted_count", 0), + "awaiting_submission_count": review_summary.get("awaiting_submission_count", 0), + "ready_for_ledger_review_count": review_summary.get("ready_for_ledger_review_count", 0), + "valid_packet_source_incomplete_count": review_summary.get("valid_packet_source_incomplete_count", 0), + "invalid_submission_count": review_summary.get("invalid_submission_count", 0), + "ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True, + "runbook_counts_as_completion": False, + "decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence", + }, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "runbook_counts_submission_as_completion": False, + }, + "items": items, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json", + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html", + }, + } + + +def list_lines(values: list[Any], empty: str) -> list[str]: + if not values: + return [f"- {empty}"] + return [f"- {value}" for value in values] + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Operator Runbook", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- runbook counts as completion: `{str(summary['runbook_counts_as_completion']).lower()}`", + f"- evidence items: `{summary['evidence_item_count']}`", + f"- pending: `{summary['pending_count']}`", + f"- awaiting submission: `{summary['awaiting_submission_count']}`", + f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`", + "", + "This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.", + "", + "## Fast Path", + "", + "1. Run the real external or human work for one evidence item.", + "2. Generate the matching submission draft.", + "3. Replace template-only fields with aggregate evidence and provenance.", + "4. Validate intake, review the queue, refresh the ledger, then run the claim guard.", + "", + "## Evidence Items", + "", + "| Evidence | Ledger | Intake | Review | Owner |", + "| --- | --- | --- | --- | --- |", + ] + for item in report["items"]: + lines.append( + f"| `{item['evidence_key']}` | `{item['ledger_status']}` | `{item['intake_readiness']}` | " + f"`{item['review_state']}` | {item['owner']} |" + ) + for item in report["items"]: + lines.extend( + [ + "", + f"## {item['label']}", + "", + f"- objective: {item['objective']}", + f"- blocking reason: {item['blocking_reason']}", + f"- submission: `{item['submission_path'] or 'missing'}`", + f"- template: `{item['template_path'] or 'missing'}`", + "", + "### Commands", + "", + ] + ) + for label, command in item["commands"].items(): + lines.append(f"- {label}: `{command}`") + lines.extend(["", "### Must Collect", ""]) + lines.extend(list_lines(item["must_collect"].get("provenance_requirements", []), "No provenance requirements listed.")) + lines.extend(["", "### Success Checks", ""]) + lines.extend(list_lines(item["must_collect"].get("success_checks", []), "No success checks listed.")) + lines.extend(["", "### Privacy Contract", ""]) + lines.extend(list_lines(item["must_collect"].get("privacy_contract", []), "No privacy contract listed.")) + lines.extend(["", "### Evidence Artifacts", ""]) + lines.extend(list_lines(item.get("evidence_artifacts", []), "No evidence artifacts listed.")) + lines.extend( + [ + "", + "## Boundary", + "", + "- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.", + "- Valid intake means ready for ledger review, not accepted evidence.", + "- The world-class ledger and claim guard remain the source of truth.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def html_list(values: list[Any], empty: str) -> str: + if not values: + return f"
  • {html_text(empty)}
  • " + return "".join(f"
  • {html_text(value)}
  • " for value in values) + + +def render_html_item(item: dict[str, Any]) -> str: + commands = "".join( + f"
  • {html_text(label.replace('_', ' '))}{html_text(command)}
  • " + for label, command in item["commands"].items() + ) + must_collect = item["must_collect"] + return f""" +
    +
    {html_text(item['category'])} · {html_text(item['review_state'])}

    {html_text(item['label'])}

    +

    {html_text(item['blocking_reason'])}

    +
    +
    Owner
    {html_text(item['owner'])}
    +
    Ledger
    {html_text(item['ledger_status'])}
    +
    Submission
    {html_text(item['submission_path'])}
    +
    +

    Commands

      {commands}
    +
    +

    Must Collect

      {html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}
    +

    Success Checks

      {html_list(must_collect.get('success_checks', []), 'No success checks listed.')}
    +

    Privacy

      {html_list(must_collect.get('privacy_contract', []), 'No privacy contract listed.')}
    +
    +
    + """.strip() + + +def render_html(report: dict[str, Any]) -> str: + summary = report["summary"] + stats = [ + ("Pending", summary["pending_count"]), + ("Awaiting", summary["awaiting_submission_count"]), + ("Ready", summary["ready_for_ledger_review_count"]), + ("Invalid", summary["invalid_submission_count"]), + ] + stat_html = "".join(f"
    {html_text(label)}{html_text(value)}
    " for label, value in stats) + item_html = "".join(render_html_item(item) for item in report["items"]) + return f""" + + + + + World-Class Operator Runbook + + + + +
    +
    + Evidence Operations +

    World-Class Operator Runbook

    +

    A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

    +
    {stat_html}
    +
    +

    Fast Path

    1. Run the real external or human work for one evidence item.
    2. Generate and fill the matching submission draft.
    3. Validate intake and inspect the submission review queue.
    4. Refresh the ledger and run the claim guard before making any completion claim.
    +

    Evidence Items

    {item_html}
    +

    Boundary

    • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
    • Valid intake means ready for ledger review, not accepted evidence.
    • The world-class ledger and claim guard remain the source of truth.
    +
    + + +""" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render an operator runbook for collecting pending world-class evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_operator_runbook.json") + parser.add_argument("--output-md", default="reports/world_class_operator_runbook.md") + parser.add_argument("--output-html", default="reports/world_class_operator_runbook.html") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_operator_runbook(skill_dir, args.generated_at, submissions_dir=submissions_dir) + outputs = { + "json": Path(args.output_json), + "markdown": Path(args.output_md), + "html": Path(args.output_html), + } + for key, path in outputs.items(): + if not path.is_absolute(): + path = skill_dir / path + path.parent.mkdir(parents=True, exist_ok=True) + report["artifacts"][key] = output_path(str(path), skill_dir) + if key == "json": + path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + elif key == "markdown": + path.write_text(render_markdown(report), encoding="utf-8") + else: + path.write_text(render_html(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index f9407b34..228d3d08 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -84,6 +84,8 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", "world_class_submission_review": "reports/world_class_submission_review.md", + "world_class_operator_runbook": "reports/world_class_operator_runbook.md", + "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", "world_class_claim_guard": "reports/world_class_claim_guard.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", @@ -123,6 +125,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"), "world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"), "world_class_submission_review": load_json(reports / "world_class_submission_review.json"), + "world_class_operator_runbook": load_json(reports / "world_class_operator_runbook.json"), "world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"), "registry": load_json(reports / "registry_audit.json"), "package_verification": load_json(reports / "package_verification.json"), diff --git a/scripts/yao.py b/scripts/yao.py index 8559ce6f..99497635 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -34,6 +34,7 @@ from yao_cli_report_commands import ( command_world_class_evidence, command_world_class_intake, command_world_class_ledger, + command_world_class_runbook, command_world_class_submission_kit, command_world_class_submission_review, ) @@ -176,6 +177,8 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_world_class_evidence_plan.py", [str(ROOT)]), run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), run_script("render_world_class_evidence_intake.py", [str(ROOT)]), + run_script("render_world_class_submission_review.py", [str(ROOT)]), + run_script("render_world_class_operator_runbook.py", [str(ROOT)]), run_script("render_world_class_claim_guard.py", [str(ROOT)]), run_script("render_benchmark_reproducibility.py", [str(ROOT)]), run_script("render_skill_os2_coverage.py", [str(ROOT)]), @@ -212,6 +215,8 @@ def command_report(args: argparse.Namespace) -> int: "world_class_evidence_plan": "reports/world_class_evidence_plan.json", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", "world_class_evidence_intake": "reports/world_class_evidence_intake.json", + "world_class_submission_review": "reports/world_class_submission_review.json", + "world_class_operator_runbook": "reports/world_class_operator_runbook.json", "world_class_claim_guard": "reports/world_class_claim_guard.json", "benchmark_reproducibility": "reports/benchmark_reproducibility.json", "skill_os2_coverage": "reports/skill_os2_coverage.json", @@ -741,6 +746,8 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_world_class_evidence_intake.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_submission_review.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_operator_runbook.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_world_class_claim_guard.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_skill_os2_coverage.py", [str(ROOT)])}, diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 164d53f7..80830387 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -231,6 +231,18 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] world_class_submission_review_cmd.add_argument("--generated-at") world_class_submission_review_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_review")) + world_class_runbook_cmd = subparsers.add_parser( + "world-class-runbook", + help="Render an operator runbook for pending world-class evidence.", + ) + world_class_runbook_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_runbook_cmd.add_argument("--submissions-dir") + world_class_runbook_cmd.add_argument("--output-json") + world_class_runbook_cmd.add_argument("--output-md") + world_class_runbook_cmd.add_argument("--output-html") + world_class_runbook_cmd.add_argument("--generated-at") + world_class_runbook_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_runbook")) + world_class_claim_guard_cmd = subparsers.add_parser( "world-class-claim-guard", help="Scan public claim surfaces for premature world-class completion claims.", diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index d62f4c60..15cfe782 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -100,6 +100,14 @@ def command_world_class_submission_review(args: argparse.Namespace) -> int: return emit_result(run_script("render_world_class_submission_review.py", cmd)) +def command_world_class_runbook(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_operator_runbook.py", cmd)) + + def command_world_class_claim_guard(args: argparse.Namespace) -> int: cmd = [resolved_skill_dir(args)] for surface in args.claim_surface: diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 96f6034d..0ad372e9 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -168,6 +168,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", @@ -186,8 +187,7 @@ "scripts/skill_report_charts.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", - "scripts/skill_report_model.py", - "scripts/sync_local_install.py" + "scripts/skill_report_model.py" ], "assets": [ "templates/basic_skill.md.j2", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 81a646bf..701cc3c6 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -80,6 +80,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 95a0c83d..dc0bc51d 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -245,6 +245,7 @@ def main() -> None: "render_world_class_evidence_ledger.py", "render_world_class_evidence_intake.py", "render_world_class_submission_review.py", + "render_world_class_operator_runbook.py", "render_world_class_claim_guard.py", "render_skill_os2_coverage.py", ]: @@ -379,6 +380,10 @@ def main() -> None: assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_submission_review.md").exists(): assert full_payload["evidence_paths"]["world_class_submission_review"] == "reports/world_class_submission_review.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_operator_runbook.md").exists(): + assert full_payload["evidence_paths"]["world_class_operator_runbook"] == "reports/world_class_operator_runbook.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_operator_runbook.html").exists(): + assert full_payload["evidence_paths"]["world_class_operator_runbook_html"] == "reports/world_class_operator_runbook.html", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_claim_guard.md").exists(): assert full_payload["evidence_paths"]["world_class_claim_guard"] == "reports/world_class_claim_guard.md", full_payload["evidence_paths"] assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"] @@ -420,12 +425,16 @@ def main() -> None: "reports/skill_os2_audit.md", }, world_class_action assert all(item["exists"] for item in world_class_action["source_refs"]), world_class_action - assert "world-class-submission-review" in world_class_action["verification_command"], world_class_action + assert "world-class-runbook" in world_class_action["verification_command"], world_class_action + assert "reports/world_class_operator_runbook.html" in world_class_action["source_fix"], world_class_action assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4, full_payload["data"]["world_class_evidence_ledger"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"] assert full_payload["data"]["world_class_submission_review"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_submission_review"]["summary"]["review_counts_submission_as_completion"] is False, full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_submission_review"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_submission_review"] + assert full_payload["data"]["world_class_operator_runbook"]["summary"]["decision"] == "collect-evidence", full_payload["data"]["world_class_operator_runbook"] + assert full_payload["data"]["world_class_operator_runbook"]["summary"]["runbook_counts_as_completion"] is False, full_payload["data"]["world_class_operator_runbook"] + assert full_payload["data"]["world_class_operator_runbook"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_operator_runbook"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["template_pass_count"] == 4, full_payload["data"]["world_class_evidence_intake"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_count"] == 4, full_payload["data"]["world_class_evidence_intake"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_ready_count"] == 0, full_payload["data"]["world_class_evidence_intake"] @@ -515,6 +524,7 @@ def main() -> None: assert "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" in html, html assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html assert "reports/world_class_evidence_intake.md" in html, html + assert "reports/world_class_operator_runbook.html" in html, html assert "reports/world_class_claim_guard.md" in html, html assert "英文完成断言、true 状态声明或中文完成态" in html, html assert "world-evidence-grid" in html, html diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py new file mode 100644 index 00000000..f628b0a5 --- /dev/null +++ b/tests/verify_world_class_operator_runbook.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_operator_runbook.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_world_class_operator_runbook" + + +def provider_submission() -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-14", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_execution_runs.json", + "kind": "aggregate-report", + "contains_raw_content": False, + "sha256": "example-only", + } + ], + "provenance": {"provider": "openai", "model": "gpt-4.1-mini", "credential_material_committed": False}, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + }, + } + + +def run_direct(*extra: str) -> dict: + proc = subprocess.run( + [sys.executable, str(SCRIPT), str(ROOT), "--generated-at", "2026-06-14", *extra], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def run_cli(*extra: str) -> dict: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("YAO_CLI_TELEMETRY_EVENTS", None) + proc = subprocess.run( + [sys.executable, str(CLI), *extra], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + return json.loads(proc.stdout) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "world_class_operator_runbook.json" + output_md = TMP / "world_class_operator_runbook.md" + output_html = TMP / "world_class_operator_runbook.html" + payload = run_direct("--output-json", str(output_json), "--output-md", str(output_md), "--output-html", str(output_html)) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "collect-evidence", summary + assert summary["evidence_item_count"] == 4, summary + assert summary["pending_count"] == 4, summary + assert summary["awaiting_submission_count"] == 4, summary + assert summary["ready_for_ledger_review_count"] == 0, summary + assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["invalid_submission_count"] == 0, summary + assert summary["ready_to_claim_world_class"] is False, summary + assert summary["runbook_counts_as_completion"] is False, summary + items = {item["evidence_key"]: item for item in payload["items"]} + assert set(items) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, items + provider = items["provider-holdout"] + assert provider["review_state"] == "awaiting-submission", provider + assert provider["source_accepted"] is False, provider + assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider + assert "world-class-intake" in provider["commands"]["validate_intake"], provider + assert "world-class-submission-review" in provider["commands"]["review_queue"], provider + assert "world-class-ledger" in provider["commands"]["refresh_ledger"], provider + assert "world-class-claim-guard" in provider["commands"]["guard_claim"], provider + assert "provider-backed model run" in provider["must_collect"]["provenance_requirements"], provider + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider["must_collect"]["success_checks"], provider + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Operator Runbook" in markdown, markdown + assert "runbook counts as completion: `false`" in markdown, markdown + assert "Valid intake means ready for ledger review, not accepted evidence." in markdown, markdown + html = output_html.read_text(encoding="utf-8") + assert "World-Class Operator Runbook" in html, html[:400] + assert "ledger and claim guard" in html, html + assert "position:sticky" in html, html + assert " None: assert yao_cli_report_commands.SCRIPT_INTERFACE == "internal-module" assert callable(yao_cli_module.command_review_studio) parser_help = yao_cli_module.build_parser().format_help() - assert "quickstart" in parser_help, parser_help - assert "review-studio" in parser_help, parser_help - assert "python-compat" in parser_help, parser_help - assert "architecture-audit" in parser_help, parser_help - assert "skill-os2-audit" in parser_help, parser_help - assert "skill-os2-coverage" in parser_help, parser_help - assert "world-class-evidence" in parser_help, parser_help - assert "world-class-ledger" in parser_help, parser_help - assert "world-class-intake" in parser_help, parser_help - assert "world-class-submission-kit" in parser_help, parser_help - assert "world-class-submission-review" in parser_help, parser_help - assert "world-class-claim-guard" in parser_help, parser_help - assert "benchmark-reproducibility" in parser_help, parser_help - assert "output-review-kit" in parser_help, parser_help - assert "telemetry-import" in parser_help, parser_help - assert "telemetry-emit" in parser_help, parser_help - assert "telemetry-hooks" in parser_help, parser_help - assert "--record-cli-telemetry" in parser_help, parser_help + expected_help = ( + "quickstart review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " + "world-class-evidence world-class-ledger world-class-intake world-class-submission-kit " + "world-class-submission-review world-class-runbook world-class-claim-guard " + "benchmark-reproducibility output-review-kit telemetry-import telemetry-emit telemetry-hooks --record-cli-telemetry" + ).split() + assert all(item in parser_help for item in expected_help), parser_help init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root)) assert init_result["ok"], init_result
    路径Path作用Role类型Type
    SKILL.mdSkill 入口文件Skill entrypoint文件file
    README.md人类可读使用说明Human-readable usage guide文件file
    agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
    manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
    references扩展指导与复用资料Extended guidance and reusable notes目录folder
    scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
    evals触发与质量检查Trigger and quality checks目录folder
    reports生成的证据与总结报告Generated evidence and overview artifacts目录folder