diff --git a/Makefile b/Makefile index be0d84c4..449e2c91 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -58,6 +58,9 @@ output-eval-check: output-execution-check: $(PYTHON) tests/verify_output_execution_runs.py +output-review-kit-check: + $(PYTHON) tests/verify_output_review_kit.py + output-review-adjudication-check: $(PYTHON) tests/verify_output_review_adjudication.py diff --git a/registry/index.json b/registry/index.json index 5205ba2b..36930d77 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "package_sha256": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index d475731c..49d442ee 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,7 +16,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "package_sha256": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda", "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index de4a7b4a..f7e50a9e 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T22:37:14Z", + "generated_at": "2026-06-13T22:55:26Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index a37a50ba..30366ec3 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,12 +4,12 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 149, - "script_file_count": 92, - "test_file_count": 57, + "python_file_count": 151, + "script_file_count": 93, + "test_file_count": 58, "internal_module_count": 19, - "cli_script_count": 74, - "command_handler_count": 33, + "cli_script_count": 75, + "command_handler_count": 34, "warn_line_threshold": 900, "block_line_threshold": 1500, "largest_file_lines": 898, @@ -27,21 +27,21 @@ }, { "path": "tests/verify_yao_cli.py", - "lines": 891, + "lines": 893, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 841, + "lines": 862, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." }, { "path": "scripts/skill_report_model.py", - "lines": 782, + "lines": 792, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -53,6 +53,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/yao_cli_parser.py", + "lines": 726, + "kind": "internal-module", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/optimize_description.py", "lines": 723, @@ -67,13 +74,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "scripts/yao_cli_parser.py", - "lines": 714, - "kind": "internal-module", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/render_review_viewer.py", "lines": 685, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 861ca2f6..c6cf541a 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,12 +5,12 @@ Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `149` -- scripts: `92` -- tests: `57` +- python files: `151` +- scripts: `93` +- tests: `58` - internal modules: `19` -- CLI scripts: `74` -- Yao CLI command handlers: `33` +- CLI scripts: `75` +- Yao CLI command handlers: `34` - largest file lines: `898` - hotspots: `0` - blockers: `0` @@ -26,13 +26,13 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `scripts/render_review_studio.py` | `898` | `cli-script` | `pass` | -| `tests/verify_yao_cli.py` | `891` | `test` | `pass` | -| `scripts/yao.py` | `841` | `cli-script` | `pass` | -| `scripts/skill_report_model.py` | `782` | `internal-module` | `pass` | +| `tests/verify_yao_cli.py` | `893` | `test` | `pass` | +| `scripts/yao.py` | `862` | `cli-script` | `pass` | +| `scripts/skill_report_model.py` | `792` | `internal-module` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | +| `scripts/yao_cli_parser.py` | `726` | `internal-module` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | -| `scripts/yao_cli_parser.py` | `714` | `internal-module` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `674` | `cli-script` | `pass` | | `scripts/skill_report_layout.py` | `653` | `internal-module` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 46b61156..40384470 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,16 +3,13 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", + "commit": "ecdb16dd90025bcca3051be4c6a1b5b723c7ec75", "git_status": { "available": true, "dirty": true, - "changed_file_count": 60, + "changed_file_count": 59, "sample": [ - " M AGENTS.md", " M Makefile", - " M README.md", - " M evidence/world_class/README.md", " M registry/index.json", " M registry/packages/yao-meta-skill.json", " M reports/adoption_drift_report.json", @@ -20,7 +17,10 @@ " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md" + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/install_simulation.json" ] }, "summary": { @@ -42,7 +42,7 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 60 + "changed_file_count": 59 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -116,7 +116,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "34fa0a084b7537cca412b4bbea4018639ceac9e98ddf83a6d25ee2dc7f7dbe4f" + "sha256": "b86a205e0eee68676a822c4f818b5ea0496c381c783043832be72050d5a1e716" }, { "label": "blind_review", @@ -129,8 +129,8 @@ "label": "review_adjudication", "path": "reports/output_review_adjudication.json", "exists": true, - "bytes": 9125, - "sha256": "5765254c430947c7e2b05883831d7db0b886d607b59f9a8a2602dcfa925f400b" + "bytes": 9495, + "sha256": "240485a721af49d5c75fe8049e9ef74b50546f685aa6346ab9e20713fb1105f4" }, { "label": "trigger_scorecard", @@ -150,22 +150,22 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 94809, - "sha256": "1d256935f965a265ae5c60e8600a081db8aebe1fdeff7068c5d08fb521fe653b" + "bytes": 95873, + "sha256": "6205af0ad3c9980eab785d7644832b1ca804d898eebcf993cced83c23b256fa4" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 19840, - "sha256": "9b8f5b17fe2711b17557366edda49a89bbf2e378576de52d7de798137dc8a421" + "bytes": 20093, + "sha256": "e5dd973d0b5c78c5e40d52727939134b8167d742cb90bea2141744c0f7d5fa33" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "c3ae3224b9339aec1e01e695835aa14113bf23979100eda2099fa9609e841c17" + "sha256": "daedc9b8047e2926221f39209d0751dac67b160c2c68fded0fcca796d9727144" }, { "label": "package_verification", @@ -179,42 +179,42 @@ "path": "reports/install_simulation.json", "exists": true, "bytes": 8758, - "sha256": "465f407ccc808169f5380550161f481ab8f0fbc20ef9ab170814fb703b05fe3b" + "sha256": "3c9306b5b1221f5951f69598c8e7104557ea17635f11c42b8b52a37cae6d22a2" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 13946, - "sha256": "340c10df7350d74cbfe3fd77461ce2cbab201198fd93fd276e7273eb6f6222e7" + "sha256": "9c22f67024a63d8646c2306ef642e06848bb443adde690e2f97573024e3a636b" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 9532, - "sha256": "ed1274b7c18cac9e683bb4a24a5ed8039d1eb582d676f0fc26cc3a33e5371a67" + "bytes": 9634, + "sha256": "7ff33ae06f00247f97fa85ca3268aaa61f672aebba75ac96fc3d932b4997daa2" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 10750, - "sha256": "e6dfdb22dc9f54255c1d4b42d36c035ffafbca4fe1d1bc6569c5e8262443f257" + "bytes": 10790, + "sha256": "2e2dc29095a05d481353fec00f5b87c3381c81ba9661fb36b03496d5e5db325e" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 12369, - "sha256": "465f8d915a869ad536c2d71ba26dc536b82ba64e6e98013e8dbd0efd1bb92883" + "bytes": 12411, + "sha256": "10ac53e66486505ff1646f5616ffa334c0e2a2ac99d9d54cf6222d2ee3868063" }, { "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, - "bytes": 8093, - "sha256": "24396b9ecb4351cb9e3dff3d69b83bb940bf1507387a086cb0a08ca8cfdfa618" + "bytes": 8179, + "sha256": "299a88eb7b47d16761d66980e01ccd8164609e344c3c98bb0b8a67b56cf68edd" } ], "missing_artifacts": [], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 55c53334..44d02ebd 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `30539bed7a29dd88c2f289dd0c115da9834dd5d9` +Commit: `ecdb16dd90025bcca3051be4c6a1b5b723c7ec75` Working tree dirty at generation: `true` ## Summary @@ -16,7 +16,7 @@ Working tree dirty at generation: `true` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `60` +- changed files at generation: `59` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `34fa0a084b75` | +| output_execution | `reports/output_execution_runs.json` | present | `b86a205e0eee` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | -| review_adjudication | `reports/output_review_adjudication.json` | present | `5765254c4309` | +| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `1d256935f965` | -| python_compatibility | `reports/python_compatibility.json` | present | `9b8f5b17fe27` | -| registry_audit | `reports/registry_audit.json` | present | `c3ae3224b933` | +| trust_report | `reports/security_trust_report.json` | present | `6205af0ad3c9` | +| python_compatibility | `reports/python_compatibility.json` | present | `e5dd973d0b5c` | +| registry_audit | `reports/registry_audit.json` | present | `daedc9b8047e` | | package_verification | `reports/package_verification.json` | present | `89dbb682f4a4` | -| install_simulation | `reports/install_simulation.json` | present | `465f407ccc80` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `340c10df7350` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `ed1274b7c18c` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `e6dfdb22dc9f` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `465f8d915a86` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `24396b9ecb43` | +| install_simulation | `reports/install_simulation.json` | present | `3c9306b5b122` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `9c22f67024a6` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `7ff33ae06f00` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `2e2dc29095a0` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `10ac53e66486` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `299a88eb7b47` | ## Reproduction Commands diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index 6d5da159..77a50459 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -166,6 +166,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -223,8 +224,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -383,7 +383,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -399,6 +399,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -485,14 +486,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -514,7 +515,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -627,7 +628,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -643,6 +644,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -729,14 +731,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -756,7 +758,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -1011,6 +1013,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -1068,8 +1071,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -1228,7 +1230,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1244,6 +1246,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -1330,14 +1333,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1359,7 +1362,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -1472,7 +1475,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1488,6 +1491,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -1574,14 +1578,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1601,7 +1605,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -1856,6 +1860,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -1913,8 +1918,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2073,7 +2077,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2089,6 +2093,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -2175,14 +2180,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2204,7 +2209,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -2310,7 +2315,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2326,6 +2331,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -2412,14 +2418,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2439,7 +2445,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -2685,6 +2691,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -2742,8 +2749,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2902,7 +2908,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2918,6 +2924,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -3004,14 +3011,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3033,7 +3040,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -3139,7 +3146,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3155,6 +3162,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -3241,14 +3249,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3268,7 +3276,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -3514,6 +3522,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -3571,8 +3580,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -3731,7 +3739,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3747,6 +3755,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -3833,14 +3842,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3862,7 +3871,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -3972,7 +3981,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3988,6 +3997,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -4074,14 +4084,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4101,7 +4111,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, diff --git a/reports/context_budget.json b/reports/context_budget.json index 8d754306..299abfd2 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1129811, + "other_text_tokens": 1145183, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1130562, - "relevant_file_count": 484, + "estimated_total_text_tokens": 1145934, + "relevant_file_count": 490, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index f2896651..1a6a3ce4 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 563, + "archive_entry_count": 567, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 724ac2c6..127764be 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.77, + "duration_ms": 27.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.86, + "duration_ms": 26.83, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.67, + "duration_ms": 28.0, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.37, + "duration_ms": 28.5, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.7, + "duration_ms": 28.58, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.96, + "duration_ms": 28.82, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.07, + "duration_ms": 28.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.42, + "duration_ms": 28.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.64, + "duration_ms": 28.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.43, + "duration_ms": 29.97, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 298a7261..4256a5b7 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.77 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 28.86 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 28.67 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.37 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.7 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.96 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.07 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.42 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.64 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.43 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.61 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.83 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.0 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.5 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.58 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.82 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.41 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.26 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.37 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 29.97 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/output_review_adjudication.json b/reports/output_review_adjudication.json index f484553c..ab0da581 100644 --- a/reports/output_review_adjudication.json +++ b/reports/output_review_adjudication.json @@ -92,6 +92,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -118,6 +119,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -144,6 +146,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -170,6 +173,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -196,6 +200,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." diff --git a/reports/output_review_adjudication.md b/reports/output_review_adjudication.md index 2fc622ce..b8a56a36 100644 --- a/reports/output_review_adjudication.md +++ b/reports/output_review_adjudication.md @@ -46,6 +46,7 @@ Expected winners stay hidden until a valid reviewer decision is recorded. #### Commands +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` - write_template: `python3 scripts/adjudicate_output_review.py --write-template` - adjudicate: `python3 scripts/yao.py output-review` - refresh_review_studio: `python3 scripts/yao.py review-studio .` @@ -72,6 +73,7 @@ Expected winners stay hidden until a valid reviewer decision is recorded. #### Commands +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` - write_template: `python3 scripts/adjudicate_output_review.py --write-template` - adjudicate: `python3 scripts/yao.py output-review` - refresh_review_studio: `python3 scripts/yao.py review-studio .` @@ -98,6 +100,7 @@ Expected winners stay hidden until a valid reviewer decision is recorded. #### Commands +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` - write_template: `python3 scripts/adjudicate_output_review.py --write-template` - adjudicate: `python3 scripts/yao.py output-review` - refresh_review_studio: `python3 scripts/yao.py review-studio .` @@ -124,6 +127,7 @@ Expected winners stay hidden until a valid reviewer decision is recorded. #### Commands +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` - write_template: `python3 scripts/adjudicate_output_review.py --write-template` - adjudicate: `python3 scripts/yao.py output-review` - refresh_review_studio: `python3 scripts/yao.py review-studio .` @@ -150,6 +154,7 @@ Expected winners stay hidden until a valid reviewer decision is recorded. #### Commands +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` - write_template: `python3 scripts/adjudicate_output_review.py --write-template` - adjudicate: `python3 scripts/yao.py output-review` - refresh_review_studio: `python3 scripts/yao.py review-studio .` diff --git a/reports/output_review_kit.json b/reports/output_review_kit.json new file mode 100644 index 00000000..196708b4 --- /dev/null +++ b/reports/output_review_kit.json @@ -0,0 +1,267 @@ +{ + "schema_version": "1.0", + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "review_contract": { + "reviewer_steps": [ + "Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.", + "Record choices in reports/output_review_decisions.json without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Run python3 scripts/yao.py output-review after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval." + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Short rationale based on the rubric, not on hidden labels." + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement." + ] + }, + "cases": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + } + ], + "failures": [] +} diff --git a/reports/output_review_kit.md b/reports/output_review_kit.md new file mode 100644 index 00000000..bc63b6c8 --- /dev/null +++ b/reports/output_review_kit.md @@ -0,0 +1,138 @@ +# Output Review Kit + +This reviewer-facing packet contains the blind A/B cases, decision fields, and command flow. It intentionally does not expose the answer key. + +## Summary + +- cases: `5` +- ready for adjudication: `0` +- pending decisions: `5` +- invalid decisions: `0` +- reviewer metadata present: `false` +- answer key hidden: `true` +- answer key path exposed: `false` + +## Review Flow + +1. Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case. +2. Record choices in reports/output_review_decisions.json without opening the answer key. +3. Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case. +4. Run python3 scripts/yao.py output-review after choices are recorded. +5. Refresh python3 scripts/yao.py review-studio . before asking for release approval. + +## Required Fields + +- `reviewer`: Human reviewer name or review group. +- `reviewed_at`: Review date or timestamp. +- `winner_variant`: A or B for every case. +- `confidence`: Optional numeric confidence from 0 to 1. +- `reason`: Short rationale based on the rubric, not on hidden labels. + +## Privacy Contract + +- The answer key is intentionally withheld from this kit. +- Do not inspect hidden labels or expected winners before decisions are recorded. +- Do not paste private user data into decision reasons. +- Pending decisions must stay pending instead of being counted as human agreement. + +## Decision States + +| Case | State | Winner | Confidence | Reason | Blocking Reason | +| --- | --- | --- | --- | --- | --- | +| `skill-package-contract` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `output-eval-expectation` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `ir-before-packaging` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `near-neighbor-boundary` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `file-backed-governed-package` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | + +## Blind Cases + +### skill-package-contract + +Prompt: Turn this repeated workflow into a reusable team skill package. + +Rubric: +- `has-entrypoint` (1.0): Output names the routeable Skill entrypoint. +- `has-interface` (1.0): Output includes neutral interface metadata. +- `has-report` (1.0): Output points reviewer to generated overview report. +- `has-resource-boundary` (1.0): Output preserves entrypoint/resource separation. + +#### Variant A + +Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/. + +#### Variant B + +I can write a prompt for that workflow and include a short checklist. + +### output-eval-expectation + +Prompt: Upgrade this production skill so we know whether the generated output is better than baseline. + +Rubric: +- `has-baseline-comparison` (1.0): Output explicitly compares with-skill and baseline outputs. +- `has-assertion-grading` (1.0): Output includes objective assertion grading. +- `has-scorecard` (1.0): Output produces a scorecard report path. +- `has-failure-taxonomy` (1.0): Output records failure taxonomy. + +#### Variant A + +Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md. + +#### Variant B + +Add more examples and run the trigger eval again. + +### ir-before-packaging + +Prompt: Package this skill for OpenAI, Claude, Agent Skills, and generic targets. + +Rubric: +- `has-ir-first` (1.0): Output requires Skill IR before packaging. +- `has-core-ir-fields` (1.0): Output lists core IR contract fields. +- `has-targets` (1.0): Output names requested runtime targets. +- `has-semantic-preservation` (1.0): Output says targets must preserve the capability contract. + +#### Variant A + +Export folders for each platform and adjust files as needed. + +#### Variant B + +Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract. + +### near-neighbor-boundary + +Prompt: I only need a one-off summary of these notes, not a reusable process. + +Rubric: +- `declines-skill` (1.0): Output refuses unnecessary skill creation for one-off work. +- `names-near-neighbor` (1.0): Output labels the request as near-neighbor instead of owned work. +- `requires-reuse-signal` (1.0): Output asks for repeat-use evidence before packaging. + +#### Variant A + +Create a SKILL.md and a reusable workflow anyway so future notes can use it. + +#### Variant B + +Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract. + +### file-backed-governed-package + +Prompt: Turn the attached release brief source into a governed skill package. + +Rubric: +- `uses-file-backed-evidence` (1.0): Output names file-backed source evidence. +- `has-governance` (1.0): Output preserves governed ownership metadata. +- `has-output-contract` (1.0): Output preserves output and rollback boundaries. +- `has-trust-and-scorecard` (1.0): Output requires trust and output scorecard artifacts. +- `does-not-invent-evidence` (1.0): Output forbids invented launch evidence. + +#### Variant A + +Draft a release announcement with the changelog, support notes, and owner name. + +#### Variant B + +Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them. diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index ecfc9c9c..290a9303 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 152, + "file_count": 154, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -166,6 +166,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/prepare_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/prepare_world_class_submission_kit.py", "ok": true, @@ -706,6 +712,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_output_risk_profile.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index f5fbea0c..d4e290ef 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-13` - decision: `pass` - target python: `3.11` -- files scanned: `152` +- files scanned: `154` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 5aa97c72..017e65f7 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,7 +21,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "package_sha256": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda", "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "package_sha256": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 0dfa22be..c83e6de4 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,7 +6,7 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926` +- Package SHA256: `33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda` - Archive SHA256: `cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e` - Install simulated: `True` diff --git a/reports/review-studio.html b/reports/review-studio.html index 1f39e1f7..8d463c05 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -434,12 +434,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

92 scripts scanned; secrets found

Py Compat0

152 files scanned for Python 3.11

Arch Debt0

898 largest lines; 33 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

69 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

563 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

93 scripts scanned; secrets found

Py Compat0

154 files scanned for Python 3.11

Arch Debt0

898 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

70 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

563 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 92 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 152 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

149 Python files; 0 hotspots; 0 blockers; largest 898 lines; 33 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

151 Python files; 0 hotspots; 0 blockers; largest 898 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -465,7 +465,7 @@

证据路径

- +
@@ -484,7 +484,7 @@

审定报告

Pair Count
5
Judgment Count
0
待审
5
Agreement Count
0
Disagreement Count
0
Invalid Decision Count
0
答案揭示
0
答案隐藏
5
Reviewer Checklist Count
5
Reviewer Checklist Pending Count
5
-

评审清单

每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

awaiting-decision · 答案隐藏

skill-package-contract

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Turn this repeated workflow into a reusable team skill package.

操作命令

  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

output-eval-expectation

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Upgrade this production skill so we know whether the generated output is better than baseline.

操作命令

  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

ir-before-packaging

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Package this skill for OpenAI, Claude, Agent Skills, and generic targets.

操作命令

  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

near-neighbor-boundary

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
I only need a one-off summary of these notes, not a reusable process.

操作命令

  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

file-backed-governed-package

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Turn the attached release brief source into a governed skill package.

操作命令

  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
+

评审清单

先打开 reports/output_review_kit.md；每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

awaiting-decision · 答案隐藏

skill-package-contract

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Turn this repeated workflow into a reusable team skill package.

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

output-eval-expectation

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Upgrade this production skill so we know whether the generated output is better than baseline.

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

ir-before-packaging

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Package this skill for OpenAI, Claude, Agent Skills, and generic targets.

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

near-neighbor-boundary

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
I only need a one-off summary of these notes, not a reusable process.

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

file-backed-governed-package

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示词
Turn the attached release brief source into a governed skill package.

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonShort rationale; do not reveal baseline or with-skill labels before adjudication.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

人工结论

没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

@@ -506,12 +506,12 @@
-

信任报告

Secret
0
脚本数
92
网络脚本
3
Help 失败
0
包体哈希
68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926
+

信任报告

Secret
0
脚本数
93
网络脚本
3
Help 失败
0
包体哈希
7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
152
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
154
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

@@ -531,8 +531,8 @@
-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

-

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

+

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
@@ -543,7 +543,7 @@

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
待补证 · external

Native Permission Enforcement

Prove at least one target or installer enforces approved high-permission capabilities at runtime.

负责人
target client or installer integrator
当前状态
native-enforced targets 0
下一步
Integrate a real client or installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
待补证 · external

Native Permission Enforcement

Prove at least one target or installer enforces approved high-permission capabilities at runtime.

负责人
target client or installer integrator
当前状态
native-enforced targets 0
下一步
Integrate a real client or installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -568,7 +568,7 @@
-

声明守卫

台账可声明
否
台账待补
4
声明面
69
违规数
0
Overclaim Guard Active
是
+

声明守卫

台账可声明
否
台账待补
4
声明面
70
违规数
0
Overclaim Guard Active
是

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

diff --git a/reports/review-studio.json b/reports/review-studio.json index d6f93622..e88be3f6 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 92 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 152 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "149 Python files; 0 hotspots; 0 blockers; largest 898 lines; 33 CLI handlers", + "detail": "151 Python files; 0 hotspots; 0 blockers; largest 898 lines; 34 CLI handlers", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -107,7 +107,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low", "evidence": "reports/adoption_drift_report.json", "link": "adoption_drift_report.md" }, @@ -381,6 +381,7 @@ "output_eval": "reports/output_quality_scorecard.md", "output_execution": "reports/output_execution_runs.md", "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_kit": "reports/output_review_kit.md", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", "benchmark_reproducibility": "reports/benchmark_reproducibility.md", @@ -426,9 +427,10 @@ "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", - "reports/output_blind_answer_key.json", + "reports/output_review_kit.md", "reports/output_review_adjudication.md", "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -846,7 +848,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 92 + "file_count": 93 }, { "path": "evals", @@ -858,10 +860,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 197 + "file_count": 199 } ], - "file_count": 353, + "file_count": 356, "folder_count": 4, "distribution": [ { @@ -886,7 +888,7 @@ }, { "label": "scripts", - "value": 92 + "value": 93 }, { "label": "evals", @@ -894,7 +896,7 @@ }, { "label": "reports", - "value": 197 + "value": 199 } ] }, @@ -1015,7 +1017,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 92 + "file_count": 93 }, { "path": "evals", @@ -1027,7 +1029,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 197 + "file_count": 199 } ], "strengths": [ @@ -1275,6 +1277,30 @@ "pair_count": 5, "answer_key_separate": true }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, "output_review_adjudication": { "ok": true, "summary": { @@ -1319,9 +1345,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 56 + "changed_file_count": 39 }, - "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", + "commit": "ecdb16dd90025bcca3051be4c6a1b5b723c7ec75", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1396,8 +1422,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 176, - "script_count": 92, + "scanned_files": 177, + "script_count": 93, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -1406,18 +1432,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 62, + "file_write_script_count": 63, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 75, + "help_smoke_checked_count": 76, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 176, - "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" + "package_hash_file_count": 177, + "package_sha256": "c54602a493296e121f3d21a17d7b78770f93f2096de94edb3ee49b1d1cb71f13" }, "skill_atlas": { "skill_count": 12, @@ -1455,7 +1481,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7", + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { @@ -1492,7 +1518,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1515,7 +1541,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 563, + "archive_entry_count": 567, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1587,7 +1613,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" + "to": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] }, @@ -3743,7 +3769,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.33, + "duration_ms": 31.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3771,7 +3797,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.24, + "duration_ms": 30.87, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3794,7 +3820,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.8, + "duration_ms": 30.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3822,7 +3848,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.45, + "duration_ms": 30.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3845,7 +3871,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.13, + "duration_ms": 30.74, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3873,7 +3899,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.51, + "duration_ms": 30.89, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3896,7 +3922,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.39, + "duration_ms": 32.63, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3923,7 +3949,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.61, + "duration_ms": 31.17, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3946,7 +3972,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.3, + "duration_ms": 30.7, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3975,7 +4001,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.55, + "duration_ms": 30.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4202,6 +4228,273 @@ } ] }, + "output_review_kit": { + "schema_version": "1.0", + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "review_contract": { + "reviewer_steps": [ + "Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.", + "Record choices in reports/output_review_decisions.json without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Run python3 scripts/yao.py output-review after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval." + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Short rationale based on the rubric, not on hidden labels." + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement." + ] + }, + "cases": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + } + ], + "failures": [] + }, "output_review_adjudication": { "schema_version": "1.0", "ok": true, @@ -4296,6 +4589,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -4322,6 +4616,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -4348,6 +4643,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -4374,6 +4670,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -4400,6 +4697,7 @@ "blind_pack_path": "reports/output_blind_review_pack.json", "decisions_path": "reports/output_review_decisions.json", "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio ." @@ -4421,7 +4719,7 @@ "skill_os2_coverage": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "item_count": 20, @@ -4542,7 +4840,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "92 scripts; secrets 0; help failures 0", + "current": "93 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -5037,7 +5335,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "target_count": 5, @@ -5202,6 +5500,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -5259,8 +5558,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -5419,7 +5717,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5435,6 +5733,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -5521,14 +5820,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5550,7 +5849,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -5663,7 +5962,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5679,6 +5978,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -5765,14 +6065,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5792,7 +6092,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -6047,6 +6347,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -6104,8 +6405,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -6264,7 +6564,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6280,6 +6580,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -6366,14 +6667,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6395,7 +6696,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -6508,7 +6809,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6524,6 +6825,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -6610,14 +6912,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6637,7 +6939,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -6892,6 +7194,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -6949,8 +7252,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -7109,7 +7411,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7125,6 +7427,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -7211,14 +7514,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7240,7 +7543,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -7346,7 +7649,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7362,6 +7665,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -7448,14 +7752,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7475,7 +7779,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -7721,6 +8025,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -7778,8 +8083,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -7938,7 +8242,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7954,6 +8258,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -8040,14 +8345,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8069,7 +8374,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -8175,7 +8480,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8191,6 +8496,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -8277,14 +8583,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8304,7 +8610,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -8550,6 +8856,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -8607,8 +8914,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -8767,7 +9073,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8783,6 +9089,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -8869,14 +9176,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8898,7 +9205,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -9008,7 +9315,7 @@ }, "file_write": { "required": true, - "script_count": 62, + "script_count": 63, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -9024,6 +9331,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -9110,14 +9418,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 75, + "checked_count": 76, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 62, + "file_write_script_count": 63, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -9137,7 +9445,7 @@ ], "capability_counts": { "network": 3, - "file_write": 62, + "file_write": 63, "subprocess": 9, "interactive": 0 }, @@ -9809,8 +10117,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 176, - "script_count": 92, + "scanned_files": 177, + "script_count": 93, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -9819,18 +10127,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 62, + "file_write_script_count": 63, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 75, + "help_smoke_checked_count": 76, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 176, - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "package_hash_file_count": 177, + "package_sha256": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b" }, "failures": [], "warnings": [], @@ -10139,6 +10447,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/prepare_output_review_kit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/prepare_world_class_submission_kit.py", "interface": "cli", @@ -11172,9 +11494,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 75, - "checked_count": 75, - "passed_count": 75, + "candidate_count": 76, + "checked_count": 76, + "passed_count": 76, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -11389,6 +11711,16 @@ "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", "stderr_excerpt": "" }, + { + "path": "scripts/prepare_output_review_kit.py", + "command": "python3 scripts/prepare_output_review_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_output_review_kit.py [-h] [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK_MD]\n [--decisions DECISIONS]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/prepare_world_class_submission_kit.py", "command": "python3 scripts/prepare_world_class_submission_kit.py --help", @@ -12067,6 +12399,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -12182,7 +12515,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 152, + "file_count": 154, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -12343,6 +12676,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/prepare_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/prepare_world_class_submission_kit.py", "ok": true, @@ -12883,6 +13222,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_output_risk_profile.py", "ok": true, @@ -13124,12 +13469,12 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 149, - "script_file_count": 92, - "test_file_count": 57, + "python_file_count": 151, + "script_file_count": 93, + "test_file_count": 58, "internal_module_count": 19, - "cli_script_count": 74, - "command_handler_count": 33, + "cli_script_count": 75, + "command_handler_count": 34, "warn_line_threshold": 900, "block_line_threshold": 1500, "largest_file_lines": 898, @@ -13147,21 +13492,21 @@ }, { "path": "tests/verify_yao_cli.py", - "lines": 891, + "lines": 893, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 841, + "lines": 862, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." }, { "path": "scripts/skill_report_model.py", - "lines": 782, + "lines": 792, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -13173,6 +13518,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/yao_cli_parser.py", + "lines": 726, + "kind": "internal-module", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/optimize_description.py", "lines": 723, @@ -13187,13 +13539,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "scripts/yao_cli_parser.py", - "lines": 714, - "kind": "internal-module", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/render_review_viewer.py", "lines": 685, @@ -13238,10 +13583,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1129714, + "other_text_tokens": 1144803, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1130465, - "relevant_file_count": 484, + "estimated_total_text_tokens": 1145554, + "relevant_file_count": 490, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -14126,6 +14471,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -15092,7 +15438,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T22:35:30Z", + "generated_at": "2026-06-13T22:54:35Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -15116,14 +15462,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -15132,7 +15478,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -15144,31 +15490,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -15177,7 +15523,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "summary": { "waiver_count": 0, "active_count": 0, @@ -15227,7 +15573,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-14 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -15257,7 +15603,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -15280,7 +15626,7 @@ "world_class_evidence_ledger": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "ledger_entry_count": 4, @@ -15374,6 +15720,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", @@ -15473,7 +15820,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -15501,7 +15848,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -15541,7 +15888,7 @@ "world_class_evidence_intake": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-14", "skill_dir": ".", "summary": { "schema_present": true, @@ -15674,6 +16021,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", @@ -15811,7 +16159,7 @@ "world_class_submission_review": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "review_item_count": 4, @@ -15953,7 +16301,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -15988,7 +16336,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 69, + "claim_surface_count": 70, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -16177,6 +16525,10 @@ "path": "reports/output_review_adjudication.md", "violation_count": 0 }, + { + "path": "reports/output_review_kit.md", + "violation_count": 0 + }, { "path": "reports/package_verification.md", "violation_count": 0 @@ -16331,7 +16683,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "package_sha256": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b", "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { @@ -16388,7 +16740,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "package_sha256": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b" } ] }, @@ -17077,7 +17429,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 563, + "archive_entry_count": 567, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -17424,7 +17776,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "to": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index d49e0649..b15f6f71 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 176, - "script_count": 92, + "scanned_files": 177, + "script_count": 93, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 62, + "file_write_script_count": 63, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 75, + "help_smoke_checked_count": 76, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 176, - "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "package_hash_file_count": 177, + "package_sha256": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda" }, "failures": [], "warnings": [], @@ -332,6 +332,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/prepare_output_review_kit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/prepare_world_class_submission_kit.py", "interface": "cli", @@ -1365,9 +1379,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 75, - "checked_count": 75, - "passed_count": 75, + "candidate_count": 76, + "checked_count": 76, + "passed_count": 76, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -1582,6 +1596,16 @@ "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", "stderr_excerpt": "" }, + { + "path": "scripts/prepare_output_review_kit.py", + "command": "python3 scripts/prepare_output_review_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_output_review_kit.py [-h] [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK_MD]\n [--decisions DECISIONS]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/prepare_world_class_submission_kit.py", "command": "python3 scripts/prepare_world_class_submission_kit.py --help", @@ -2260,6 +2284,7 @@ "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 0807117c..a772e7d9 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `176` -- Scripts: `92` +- Scanned files: `177` +- Scripts: `93` - Internal script modules: `17` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `62` +- File-write scripts: `63` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `75` +- CLI help smoke checked: `76` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `176` -- Package SHA256: `68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926` +- Package hash files: `177` +- Package SHA256: `33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `75` -- Passed scripts: `75` +- Checked scripts: `76` +- Passed scripts: `76` - Failed scripts: `none` ## Script Surface @@ -83,6 +83,7 @@ | scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/prepare_output_review_kit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/prepare_world_class_submission_kit.py | cli | True | True | True | False | False | True | False | Prepares editable world-class evidence intake packets without counting drafts as accepted evidence. | | scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/promotion_checker.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 34174cf6..9049679b 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -621,7 +621,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布353项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布356项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 73812f7f..3e4d4363 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -16,9 +16,10 @@ "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", - "reports/output_blind_answer_key.json", + "reports/output_review_kit.md", "reports/output_review_adjudication.md", "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -436,7 +437,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 92 + "file_count": 93 }, { "path": "evals", @@ -448,10 +449,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 197 + "file_count": 199 } ], - "file_count": 353, + "file_count": 356, "folder_count": 4, "distribution": [ { @@ -476,7 +477,7 @@ }, { "label": "scripts", - "value": 92 + "value": 93 }, { "label": "evals", @@ -484,7 +485,7 @@ }, { "label": "reports", - "value": 197 + "value": 199 } ] }, @@ -605,7 +606,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 92 + "file_count": 93 }, { "path": "evals", @@ -617,7 +618,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 197 + "file_count": 199 } ], "strengths": [ @@ -865,6 +866,30 @@ "pair_count": 5, "answer_key_separate": true }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, "output_review_adjudication": { "ok": true, "summary": { @@ -909,9 +934,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 56 + "changed_file_count": 59 }, - "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", + "commit": "ecdb16dd90025bcca3051be4c6a1b5b723c7ec75", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -986,8 +1011,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 176, - "script_count": 92, + "scanned_files": 177, + "script_count": 93, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -996,18 +1021,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 62, + "file_write_script_count": 63, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 75, + "help_smoke_checked_count": 76, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 176, - "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" + "package_hash_file_count": 177, + "package_sha256": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b" }, "skill_atlas": { "skill_count": 12, @@ -1045,7 +1070,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7", + "package_sha256": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b", "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { @@ -1082,7 +1107,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1105,7 +1130,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 563, + "archive_entry_count": 567, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1177,7 +1202,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" + "to": "7645bfc56829c3d6888e792b170b2e734d041c930c31f7f285c0d59e49d7de1b" } ] }, @@ -1349,8 +1374,9 @@ "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Open reports/output_blind_review_pack.md and choose A or B for each pair without opening the answer key.", "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -1365,6 +1391,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", @@ -1556,6 +1583,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 3bcbf5c8..eaef5d12 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -85,6 +85,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index fa980275..7adf5276 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 92; help failures 0", + "current": "secrets 0; scripts 93; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 6fe6d1a6..e201d2c8 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-13` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 20; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 92; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 93; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 79a915dc..a5c801b7 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -122,7 +122,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "92 scripts; secrets 0; help failures 0", + "current": "93 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index ae1c6a42..7b0b8797 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -21,7 +21,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 92 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 93 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 563; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 1be1718b..4ffd60de 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -75,7 +75,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" + "to": "33ca97e8df3e3c75228595ab8bd06d65c697ef708723a0a53637bd8f22bbcfda" } ] }, diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 3aa27bd7..114c8f78 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -6,7 +6,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 69, + "claim_surface_count": 70, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -195,6 +195,10 @@ "path": "reports/output_review_adjudication.md", "violation_count": 0 }, + { + "path": "reports/output_review_kit.md", + "violation_count": 0 + }, { "path": "reports/package_verification.md", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index 1c4e1906..61e05981 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -7,7 +7,7 @@ Generated at: `2026-06-13` - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `69` +- claim surfaces scanned: `70` - violations: `0` - overclaim guard active: `true` diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 84e91347..4ef06208 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -134,6 +134,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index add660a2..62572db3 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -107,6 +107,7 @@ This report validates the intake contract for human and external evidence. A val - reports/skill_os2_audit.json item human-adjudication status becomes pass - evidence_artifacts: - reports/output_blind_review_pack.md + - reports/output_review_kit.md - reports/output_review_decisions.json - reports/output_review_adjudication.json - reports/output_review_adjudication.md diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index cec9e6a3..fa15316f 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -95,6 +95,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index 85dfd242..6a275e23 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -58,8 +58,9 @@ "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Open reports/output_blind_review_pack.md and choose A or B for each pair without opening the answer key.", "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -74,6 +75,7 @@ ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 8be81391..6dde9c64 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -63,8 +63,9 @@ This report is an execution plan for the remaining world-class evidence gaps. It ### Runbook +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. - `python3 scripts/adjudicate_output_review.py --write-template` -- Open reports/output_blind_review_pack.md and choose A or B for each pair without opening the answer key. - Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. - `python3 scripts/yao.py output-review` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` @@ -81,6 +82,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It ### Evidence Artifacts - `reports/output_blind_review_pack.md` +- `reports/output_review_kit.md` - `reports/output_review_decisions.json` - `reports/output_review_adjudication.json` - `reports/output_review_adjudication.md` diff --git a/scripts/adjudicate_output_review.py b/scripts/adjudicate_output_review.py index b4e8ec8c..6974f0c8 100644 --- a/scripts/adjudicate_output_review.py +++ b/scripts/adjudicate_output_review.py @@ -245,6 +245,7 @@ def build_reviewer_checklist( "blind_pack_path": display_path(blind_pack_path), "decisions_path": display_path(decisions_path), "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio .", diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 6167e87a..8bfacaf6 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -40,6 +40,7 @@ DEFAULT_TARGETS = [ "compiler-check", "output-eval-check", "output-execution-check", + "output-review-kit-check", "output-review-adjudication-check", "runtime-conformance-check", "runtime-permission-check", diff --git a/scripts/prepare_output_review_kit.py b/scripts/prepare_output_review_kit.py new file mode 100644 index 00000000..283568c3 --- /dev/null +++ b/scripts/prepare_output_review_kit.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Prepare a reviewer-facing blind A/B output review kit.""" + +import argparse +import json +from pathlib import Path +from typing import Any + +from adjudicate_output_review import ( + build_decision_template, + confidence_value, + decision_index, + display_path, + load_json, + normalize_variant, +) + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_BLIND_PACK_JSON = ROOT / "reports" / "output_blind_review_pack.json" +DEFAULT_BLIND_PACK_MD = ROOT / "reports" / "output_blind_review_pack.md" +DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json" +DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_kit.json" +DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_kit.md" + + +def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]: + if not path.exists(): + return {"schema_version": "1.0", "decisions": []}, [] + return load_json(path) + + +def pairs_from_pack(blind_pack: dict[str, Any]) -> list[dict[str, Any]]: + pairs = blind_pack.get("pairs", []) + if not isinstance(pairs, list): + return [] + return [item for item in pairs if isinstance(item, dict) and item.get("case_id")] + + +def decision_state(decision: dict[str, Any] | None) -> dict[str, Any]: + if not decision: + return { + "status": "awaiting-decision", + "winner_variant_recorded": False, + "confidence_recorded": False, + "reason_recorded": False, + "blocking_reason": "No reviewer choice has been recorded yet.", + } + winner = normalize_variant(decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", "")))) + confidence, confidence_failure = confidence_value(decision.get("confidence")) + reason = str(decision.get("reason", "")).strip() + if not winner and confidence is None and not reason: + return { + "status": "awaiting-decision", + "winner_variant_recorded": False, + "confidence_recorded": False, + "reason_recorded": False, + "blocking_reason": "Decision template exists but this row is still blank.", + } + failures = [] + if winner not in {"A", "B"}: + failures.append("winner_variant must be A or B") + if confidence_failure: + failures.append(confidence_failure) + if not reason: + failures.append("reason should explain the reviewer rationale") + if failures: + return { + "status": "needs-fix", + "winner_variant_recorded": winner in {"A", "B"}, + "confidence_recorded": confidence is not None, + "reason_recorded": bool(reason), + "blocking_reason": "; ".join(failures), + } + return { + "status": "ready-for-adjudication", + "winner_variant_recorded": True, + "confidence_recorded": confidence is not None, + "reason_recorded": True, + "blocking_reason": "Reviewer choice is complete enough for adjudication.", + } + + +def build_cases(blind_pack: dict[str, Any], decisions: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + cases = [] + for pair in pairs_from_pack(blind_pack): + case_id = str(pair.get("case_id", "")) + rubric = [] + for item in pair.get("rubric", []) if isinstance(pair.get("rubric", []), list) else []: + if isinstance(item, dict): + rubric.append( + { + "id": str(item.get("id", "")), + "description": str(item.get("description", "")), + "weight": float(item.get("weight", 1) or 0), + } + ) + cases.append( + { + "case_id": case_id, + "prompt": str(pair.get("prompt", "")), + "input_files": pair.get("input_files", []) if isinstance(pair.get("input_files", []), list) else [], + "review_instruction": str(pair.get("review_instruction", "")), + "rubric": rubric, + "variant_a": { + "blind_id": str(pair.get("variant_a", {}).get("blind_id", "")) if isinstance(pair.get("variant_a"), dict) else "", + "output": str(pair.get("variant_a", {}).get("output", "")) if isinstance(pair.get("variant_a"), dict) else "", + }, + "variant_b": { + "blind_id": str(pair.get("variant_b", {}).get("blind_id", "")) if isinstance(pair.get("variant_b"), dict) else "", + "output": str(pair.get("variant_b", {}).get("output", "")) if isinstance(pair.get("variant_b"), dict) else "", + }, + "decision_state": decision_state(decisions.get(case_id)), + } + ) + return cases + + +def build_summary(cases: list[dict[str, Any]], failures: list[str], reviewer: str, reviewed_at: str) -> dict[str, Any]: + status_counts: dict[str, int] = {} + for case in cases: + status = str(case.get("decision_state", {}).get("status", "awaiting-decision")) + status_counts[status] = status_counts.get(status, 0) + 1 + ready_count = status_counts.get("ready-for-adjudication", 0) + pending_count = status_counts.get("awaiting-decision", 0) + invalid_count = status_counts.get("needs-fix", 0) + return { + "case_count": len(cases), + "ready_for_adjudication_count": ready_count, + "pending_decision_count": pending_count, + "invalid_decision_count": invalid_count, + "reviewer_metadata_present": bool(str(reviewer).strip() and str(reviewed_at).strip()), + "answer_key_hidden": True, + "answer_key_path_exposed": False, + "ready_to_run_adjudication": bool(cases) and ready_count == len(cases) and invalid_count == 0, + "failure_count": len(failures), + } + + +def build_contract(blind_pack_md: Path, decisions_path: Path) -> dict[str, Any]: + return { + "reviewer_steps": [ + f"Open {display_path(blind_pack_md)} or this kit and compare Variant A vs Variant B for each case.", + f"Record choices in {display_path(decisions_path)} without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Run python3 scripts/yao.py output-review after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval.", + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Short rationale based on the rubric, not on hidden labels.", + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement.", + ], + } + + +def render_rubric(rubric: list[dict[str, Any]]) -> list[str]: + if not rubric: + return ["- No rubric items found."] + return [f"- `{item['id']}` ({item['weight']}): {item['description']}" for item in rubric] + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + contract = payload["review_contract"] + lines = [ + "# Output Review Kit", + "", + "This reviewer-facing packet contains the blind A/B cases, decision fields, and command flow. It intentionally does not expose the answer key.", + "", + "## Summary", + "", + f"- cases: `{summary['case_count']}`", + f"- ready for adjudication: `{summary['ready_for_adjudication_count']}`", + f"- pending decisions: `{summary['pending_decision_count']}`", + f"- invalid decisions: `{summary['invalid_decision_count']}`", + f"- reviewer metadata present: `{str(summary['reviewer_metadata_present']).lower()}`", + f"- answer key hidden: `{str(summary['answer_key_hidden']).lower()}`", + f"- answer key path exposed: `{str(summary['answer_key_path_exposed']).lower()}`", + "", + "## Review Flow", + "", + ] + lines.extend(f"{index}. {step}" for index, step in enumerate(contract["reviewer_steps"], start=1)) + lines.extend(["", "## Required Fields", ""]) + lines.extend(f"- `{key}`: {value}" for key, value in contract["required_fields"].items()) + lines.extend(["", "## Privacy Contract", ""]) + lines.extend(f"- {item}" for item in contract["privacy_contract"]) + lines.extend( + [ + "", + "## Decision States", + "", + "| Case | State | Winner | Confidence | Reason | Blocking Reason |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for case in payload["cases"]: + state = case["decision_state"] + lines.append( + f"| `{case['case_id']}` | `{state['status']}` | `{str(state['winner_variant_recorded']).lower()}` | " + f"`{str(state['confidence_recorded']).lower()}` | `{str(state['reason_recorded']).lower()}` | {state['blocking_reason']} |" + ) + lines.extend(["", "## Blind Cases", ""]) + for case in payload["cases"]: + lines.extend( + [ + f"### {case['case_id']}", + "", + f"Prompt: {case['prompt']}", + "", + "Rubric:", + *render_rubric(case["rubric"]), + "", + "#### Variant A", + "", + case["variant_a"]["output"] or "_No output found._", + "", + "#### Variant B", + "", + case["variant_b"]["output"] or "_No output found._", + "", + ] + ) + if payload.get("failures"): + lines.extend(["## Failures", ""]) + lines.extend(f"- {failure}" for failure in payload["failures"]) + return "\n".join(lines).strip() + "\n" + + +def prepare_output_review_kit( + blind_pack_json: Path, + blind_pack_md: Path, + decisions_path: Path, + output_json: Path, + output_md: Path, + write_template: bool = False, +) -> dict[str, Any]: + blind_pack, failures = load_json(blind_pack_json) + decisions_payload, decision_failures = load_optional_decisions(decisions_path) + failures.extend(decision_failures) + template_written = False + if write_template and blind_pack and not decisions_path.exists(): + decisions_path.parent.mkdir(parents=True, exist_ok=True) + decisions_path.write_text( + json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + template_written = True + decisions_payload, decision_failures = load_optional_decisions(decisions_path) + failures.extend(decision_failures) + decisions, index_failures = decision_index(decisions_payload) + failures.extend(index_failures) + case_ids = {str(pair.get("case_id", "")) for pair in pairs_from_pack(blind_pack)} + for case_id in decisions: + if case_id not in case_ids: + failures.append(f"decision references unknown case_id: {case_id}") + cases = build_cases(blind_pack, decisions) + summary = build_summary(cases, failures, str(decisions_payload.get("reviewer", "")), str(decisions_payload.get("reviewed_at", ""))) + payload = { + "schema_version": "1.0", + "ok": not failures, + "summary": summary, + "artifacts": { + "reviewer_kit_json": display_path(output_json), + "reviewer_kit_markdown": display_path(output_md), + "blind_pack_json": display_path(blind_pack_json), + "blind_pack_markdown": display_path(blind_pack_md), + "decisions": display_path(decisions_path), + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md", + }, + "template_written": template_written, + "review_contract": build_contract(blind_pack_md, decisions_path), + "cases": cases, + "failures": failures, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare a reviewer-facing blind A/B output review kit.") + parser.add_argument("--blind-pack-json", default=str(DEFAULT_BLIND_PACK_JSON)) + parser.add_argument("--blind-pack-md", default=str(DEFAULT_BLIND_PACK_MD)) + parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS)) + parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD)) + parser.add_argument("--write-template", action="store_true") + args = parser.parse_args() + + payload = prepare_output_review_kit( + Path(args.blind_pack_json).resolve(), + Path(args.blind_pack_md).resolve(), + Path(args.decisions).resolve(), + Path(args.output_json).resolve(), + Path(args.output_md).resolve(), + write_template=args.write_template, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py index 793d3118..a4b75187 100644 --- a/scripts/render_world_class_evidence_plan.py +++ b/scripts/render_world_class_evidence_plan.py @@ -41,8 +41,9 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { "owner": "human reviewer", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Open reports/output_blind_review_pack.md and choose A or B for each pair without opening the answer key.", "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -55,6 +56,7 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { ], "evidence_artifacts": [ "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index d12e784f..b7f9ef36 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -63,6 +63,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "output_eval": "reports/output_quality_scorecard.md", "output_execution": "reports/output_execution_runs.md", "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_kit": "reports/output_review_kit.md", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", "benchmark_reproducibility": "reports/benchmark_reproducibility.md", @@ -103,6 +104,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "output_quality": load_json(reports / "output_quality_scorecard.json"), "output_execution": load_json(reports / "output_execution_runs.json"), "output_blind_review": load_json(reports / "output_blind_review_pack.json"), + "output_review_kit": load_json(reports / "output_review_kit.json"), "output_review_adjudication": load_json(reports / "output_review_adjudication.json"), "skill_os2_coverage": load_json(reports / "skill_os2_coverage.json"), "compiled_targets": load_json(reports / "compiled_targets.json"), @@ -136,6 +138,7 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: output = data["output_quality"].get("summary", {}) output_execution = data["output_execution"].get("summary", {}) output_blind = data["output_blind_review"].get("summary", {}) + output_review_kit = data["output_review_kit"].get("summary", {}) output_review = data["output_review_adjudication"].get("summary", {}) blueprint = data["skill_os2_coverage"].get("summary", {}) compiled = data["compiled_targets"].get("summary", {}) @@ -184,6 +187,11 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: "value": str(output_blind.get("pair_count", 0)), "detail": "review pairs hide baseline vs with-skill labels", }, + { + "label": "Review Kit", + "value": f"{output_review_kit.get('ready_for_adjudication_count', 0)}/{output_review_kit.get('case_count', 0)}", + "detail": f"pending {output_review_kit.get('pending_decision_count', 0)}; answer key hidden", + }, { "label": "Review A/B", "value": f"{output_review.get('judgment_count', 0)}/{output_review.get('pair_count', 0)}", diff --git a/scripts/review_studio_output_review.py b/scripts/review_studio_output_review.py index 1fe4478c..0a10f386 100644 --- a/scripts/review_studio_output_review.py +++ b/scripts/review_studio_output_review.py @@ -78,7 +78,7 @@ def render_output_review_section(adjudication: dict[str, Any]) -> str: return ( "
" "

评审清单

" - "

每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

" + "

先打开 reports/output_review_kit.md；每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

" f"{render_output_review_checklist(adjudication)}" "
" ) diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index efab2023..d72eb6f2 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -475,6 +475,7 @@ def build_report_model(skill_dir: Path) -> dict: output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json") output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json") output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json") + output_review_kit = load_json(skill_dir / "reports" / "output_review_kit.json") output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json") benchmark_reproducibility = load_json(skill_dir / "reports" / "benchmark_reproducibility.json") conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") @@ -550,8 +551,11 @@ def build_report_model(skill_dir: Path) -> dict: if (skill_dir / "reports" / "output_blind_answer_key.json").exists(): insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 deliverables.insert(insert_after, "reports/output_blind_answer_key.json") + if (skill_dir / "reports" / "output_review_kit.md").exists(): + insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_review_kit.md") if (skill_dir / "reports" / "output_review_adjudication.md").exists(): - insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + insert_after = deliverables.index("reports/output_review_kit.md") + 1 if "reports/output_review_kit.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 deliverables.insert(insert_after, "reports/output_review_adjudication.md") if (skill_dir / "reports" / "benchmark_reproducibility.md").exists(): insert_after = deliverables.index("reports/output_review_adjudication.md") + 1 if "reports/output_review_adjudication.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 @@ -680,6 +684,12 @@ def build_report_model(skill_dir: Path) -> dict: "pair_count": output_blind_review.get("summary", {}).get("pair_count", 0), "answer_key_separate": output_blind_review.get("summary", {}).get("answer_key_separate", False), }, + "output_review_kit": { + "ok": output_review_kit.get("ok", False), + "summary": output_review_kit.get("summary", {}), + "artifacts": output_review_kit.get("artifacts", {}), + "failures": output_review_kit.get("failures", []), + }, "output_review_adjudication": { "ok": output_review_adjudication.get("ok", False), "summary": output_review_adjudication.get("summary", {}), diff --git a/scripts/yao.py b/scripts/yao.py index c8202bd3..b60f2b0b 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -167,6 +167,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("compile_skill.py", [str(ROOT)]), run_script("run_output_eval.py", []), run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]), + run_script("prepare_output_review_kit.py", []), run_script("adjudicate_output_review.py", []), run_script("render_adoption_drift_report.py", [str(ROOT)]), run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), @@ -201,6 +202,7 @@ def command_report(args: argparse.Namespace) -> int: "system_model": "reports/system-model.json", "compiled_targets": "reports/compiled_targets.json", "output_execution": "reports/output_execution_runs.json", + "output_review_kit": "reports/output_review_kit.json", "output_review_adjudication": "reports/output_review_adjudication.json", "adoption_drift": "reports/adoption_drift_report.json", "telemetry_hooks": "reports/telemetry_hook_recipes.json", @@ -501,6 +503,25 @@ def command_output_execution(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_output_review_kit(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.write_template: + cmd.append("--write-template") + result = run_script("prepare_output_review_kit.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_output_review(args: argparse.Namespace) -> int: cmd = [] if args.blind_pack: diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 8b3977b7..44965208 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -587,6 +587,18 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] output_eval_cmd.add_argument("--blind-answer-key-json") output_eval_cmd.set_defaults(func=_handler(command_handlers, "command_output_eval")) + output_review_kit_cmd = subparsers.add_parser( + "output-review-kit", + help="Prepare a reviewer-facing blind A/B output review kit without exposing the answer key.", + ) + output_review_kit_cmd.add_argument("--blind-pack-json") + output_review_kit_cmd.add_argument("--blind-pack-md") + output_review_kit_cmd.add_argument("--decisions") + output_review_kit_cmd.add_argument("--output-json") + output_review_kit_cmd.add_argument("--output-md") + output_review_kit_cmd.add_argument("--write-template", action="store_true") + output_review_kit_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_kit")) + output_execution_cmd = subparsers.add_parser( "output-exec", help="Record output-eval execution evidence, timing, and token usage.", diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 67f3066b..96f6034d 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -129,6 +129,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", @@ -186,8 +187,7 @@ "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", - "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py" + "scripts/sync_local_install.py" ], "assets": [ "templates/basic_skill.md.j2", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index c481c6bc..81a646bf 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -41,6 +41,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", diff --git a/tests/verify_output_review_kit.py b/tests/verify_output_review_kit.py new file mode 100644 index 00000000..ae6e265d --- /dev/null +++ b/tests/verify_output_review_kit.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" +REVIEW_KIT = ROOT / "scripts" / "prepare_output_review_kit.py" +CLI = ROOT / "scripts" / "yao.py" + + +def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def assert_no_answer_key_leak(text: str) -> None: + forbidden = [ + "output_blind_answer_key", + "expected_winner", + "variant_a_role", + "variant_b_role", + "score_winner_role", + ] + for needle in forbidden: + assert needle not in text, f"review kit leaked hidden field {needle!r}" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_review_kit" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + scorecard_json = tmp_root / "output_quality_scorecard.json" + scorecard_md = tmp_root / "output_quality_scorecard.md" + blind_pack_json = tmp_root / "output_blind_review_pack.json" + blind_pack_md = tmp_root / "output_blind_review_pack.md" + answer_key_json = tmp_root / "output_blind_answer_key.json" + + run( + [ + str(OUTPUT_EVAL), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(scorecard_json), + "--output-md", + str(scorecard_md), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--blind-answer-key-json", + str(answer_key_json), + ] + ) + + decisions_path = tmp_root / "output_review_decisions.json" + kit_json = tmp_root / "output_review_kit.json" + kit_md = tmp_root / "output_review_kit.md" + pending_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(kit_json), + "--output-md", + str(kit_md), + "--write-template", + ] + ) + pending_payload = json.loads(pending_proc.stdout) + assert pending_payload["ok"], pending_payload + assert pending_payload["template_written"], pending_payload + assert pending_payload["summary"]["case_count"] == 5, pending_payload + assert pending_payload["summary"]["pending_decision_count"] == 5, pending_payload + assert pending_payload["summary"]["ready_for_adjudication_count"] == 0, pending_payload + assert pending_payload["summary"]["answer_key_hidden"] is True, pending_payload + assert pending_payload["summary"]["answer_key_path_exposed"] is False, pending_payload + assert pending_payload["summary"]["ready_to_run_adjudication"] is False, pending_payload + assert "answer_key" not in pending_payload["artifacts"], pending_payload["artifacts"] + assert len(pending_payload["cases"]) == 5, pending_payload["cases"] + first_case = pending_payload["cases"][0] + assert first_case["variant_a"]["output"], first_case + assert first_case["variant_b"]["output"], first_case + assert first_case["decision_state"]["status"] == "awaiting-decision", first_case + assert decisions_path.exists(), decisions_path + + kit_json_text = kit_json.read_text(encoding="utf-8") + kit_md_text = kit_md.read_text(encoding="utf-8") + assert "# Output Review Kit" in kit_md_text, kit_md_text + assert "answer key path exposed: `false`" in kit_md_text, kit_md_text + assert "Variant A" in kit_md_text and "Variant B" in kit_md_text, kit_md_text + assert "python3 scripts/yao.py output-review" in kit_md_text, kit_md_text + assert_no_answer_key_leak(kit_json_text) + assert_no_answer_key_leak(kit_md_text) + + cli_proc = run( + [ + str(CLI), + "output-review-kit", + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(tmp_root / "cli_output_review_kit.json"), + "--output-md", + str(tmp_root / "cli_output_review_kit.md"), + ] + ) + cli_payload = json.loads(cli_proc.stdout) + assert cli_payload["ok"], cli_payload + assert cli_payload["summary"]["case_count"] == 5, cli_payload + assert cli_payload["summary"]["answer_key_hidden"] is True, cli_payload + assert "answer_key" not in cli_payload["artifacts"], cli_payload["artifacts"] + + template = json.loads(decisions_path.read_text(encoding="utf-8")) + filled = { + "schema_version": "1.0", + "reviewer": "Yao QA", + "reviewed_at": "2026-06-14", + "decisions": [ + { + "case_id": item["case_id"], + "winner_variant": "A", + "confidence": 0.8, + "reason": "Variant A is clearer against the visible rubric.", + } + for item in template["decisions"] + ], + } + decisions_path.write_text(json.dumps(filled, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + ready_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(tmp_root / "ready_output_review_kit.json"), + "--output-md", + str(tmp_root / "ready_output_review_kit.md"), + ] + ) + ready_payload = json.loads(ready_proc.stdout) + assert ready_payload["ok"], ready_payload + assert ready_payload["summary"]["ready_for_adjudication_count"] == 5, ready_payload + assert ready_payload["summary"]["pending_decision_count"] == 0, ready_payload + assert ready_payload["summary"]["invalid_decision_count"] == 0, ready_payload + assert ready_payload["summary"]["reviewer_metadata_present"] is True, ready_payload + assert ready_payload["summary"]["ready_to_run_adjudication"] is True, ready_payload + assert all(case["decision_state"]["status"] == "ready-for-adjudication" for case in ready_payload["cases"]), ready_payload + + invalid = dict(filled) + invalid["decisions"] = [dict(filled["decisions"][0], winner_variant="C", reason="")] + invalid_path = tmp_root / "invalid_output_review_decisions.json" + invalid_path.write_text(json.dumps(invalid, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + invalid_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(invalid_path), + "--output-json", + str(tmp_root / "invalid_output_review_kit.json"), + "--output-md", + str(tmp_root / "invalid_output_review_kit.md"), + ] + ) + invalid_payload = json.loads(invalid_proc.stdout) + assert invalid_payload["ok"], invalid_payload + assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload + assert invalid_payload["summary"]["answer_key_hidden"] is True, invalid_payload + assert invalid_payload["cases"][0]["decision_state"]["status"] == "needs-fix", invalid_payload["cases"][0] + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index fa40b8bf..3b01190a 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -89,6 +89,7 @@ def main() -> None: assert "world-class-submission-review" in parser_help, parser_help assert "world-class-claim-guard" in parser_help, parser_help assert "benchmark-reproducibility" in parser_help, parser_help + assert "output-review-kit" in parser_help, parser_help assert "telemetry-import" in parser_help, parser_help assert "telemetry-emit" in parser_help, parser_help assert "telemetry-hooks" in parser_help, parser_help @@ -757,6 +758,7 @@ def main() -> None: assert "prompt_quality_profile" in report_result["payload"]["artifacts"], report_result assert "compiled_targets" in report_result["payload"]["artifacts"], report_result assert "output_execution" in report_result["payload"]["artifacts"], report_result + assert "output_review_kit" in report_result["payload"]["artifacts"], report_result assert "output_review_adjudication" in report_result["payload"]["artifacts"], report_result assert "adoption_drift" in report_result["payload"]["artifacts"], report_result assert "review_annotations" in report_result["payload"]["artifacts"], report_result
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder