diff --git a/registry/index.json b/registry/index.json index f7781cae..4642cce3 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183" + "package_sha256": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index da5592c1..fbe24282 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183", - "archive_sha256": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4" + "package_sha256": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75", + "archive_sha256": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4", + "archive_sha256": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 80c0c51f..eecf4d9c 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,12 +4,12 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 187, - "script_file_count": 118, + "python_file_count": 188, + "script_file_count": 119, "test_file_count": 69, "internal_module_count": 36, - "cli_script_count": 84, - "command_handler_count": 67, + "cli_script_count": 85, + "command_handler_count": 68, "entrypoint_command_handler_count": 18, "command_module_count": 6, "warn_line_threshold": 900, @@ -30,16 +30,16 @@ "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", + "path": "scripts/yao_cli_parser.py", + "lines": 870, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/yao_cli_parser.py", - "lines": 854, - "kind": "internal-module", + "path": "scripts/render_evidence_consistency.py", + "lines": 859, + "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, @@ -116,16 +116,16 @@ "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", + "path": "scripts/yao_cli_parser.py", + "lines": 870, + "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/yao_cli_parser.py", - "lines": 854, - "kind": "internal-module", + "path": "scripts/render_evidence_consistency.py", + "lines": 859, + "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 55ce5cc8..3ee68733 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,12 +5,12 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `187` -- scripts: `118` +- python files: `188` +- scripts: `119` - tests: `69` - internal modules: `36` -- CLI scripts: `84` -- Yao CLI command handlers: `67` +- CLI scripts: `85` +- Yao CLI command handlers: `68` - entrypoint command handlers: `18` - command modules: `6` - largest file lines: `899` @@ -30,8 +30,8 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | -| `scripts/yao_cli_parser.py` | `854` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_world_class_evidence_intake.py` | `795` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | @@ -46,8 +46,8 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | +| `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | `pass` | -| `scripts/yao_cli_parser.py` | `854` | `internal-module` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | | `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` | | `tests/verify_world_class_evidence_intake.py` | `795` | `test` | `pass` | diff --git a/reports/install_simulation.json b/reports/install_simulation.json index d6b1fef3..2842ba02 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 640, + "archive_entry_count": 641, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/package_verification.json b/reports/package_verification.json index 12d697cc..1cf4acb7 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4", - "archive_entry_count": 640, + "archive_sha256": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae", + "archive_entry_count": 641, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 4fd0f4c7..e13fb828 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4` +- Archive SHA256: `3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 8c764691..42aa5b65 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 190, + "file_count": 191, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -154,6 +154,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/import_output_review_decisions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/import_telemetry_events.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 8f94f6bf..0b1c48fa 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `190` +- files scanned: `191` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 223a8727..f2ebe1f4 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183", - "archive_sha256": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4" + "package_sha256": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75", + "archive_sha256": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4", + "archive_sha256": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183" + "package_sha256": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 02633d03..f3d035e5 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183` -- Archive SHA256: `7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4` +- Package SHA256: `0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75` +- Archive SHA256: `3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 7e3ab736..0ac60640 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 206, - "script_count": 118, + "scanned_files": 207, + "script_count": 119, "internal_module_count": 33, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 72, + "file_write_script_count": 73, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 85, + "help_smoke_checked_count": 86, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 206, - "package_sha256": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183" + "package_hash_file_count": 207, + "package_sha256": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75" }, "failures": [], "warnings": [], @@ -304,6 +304,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/import_output_review_decisions.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Imports human blind A/B reviewer decisions into the canonical output-review decision file.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/import_telemetry_events.py", "interface": "cli", @@ -1729,9 +1743,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 85, - "checked_count": 85, - "passed_count": 85, + "candidate_count": 86, + "checked_count": 86, + "passed_count": 86, "failed_count": 0, "skipped_count": 33, "failed_scripts": [], @@ -1896,6 +1910,16 @@ "stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest", "stderr_excerpt": "" }, + { + "path": "scripts/import_output_review_decisions.py", + "command": "python3 scripts/import_output_review_decisions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: import_output_review_decisions.py [-h] --input INPUT\n [--format {auto,json,jsonl,csv}]\n [--blind-pack BLIND_PACK]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/import_telemetry_events.py", "command": "python3 scripts/import_telemetry_events.py --help", @@ -2787,6 +2811,7 @@ "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/prepare_output_review_kit.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 255a819b..db170a6e 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `206` -- Scripts: `118` +- Scanned files: `207` +- Scripts: `119` - Internal script modules: `33` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `72` +- File-write scripts: `73` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `85` +- CLI help smoke checked: `86` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `206` -- Package SHA256: `dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183` +- Package hash files: `207` +- Package SHA256: `0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `85` -- Passed scripts: `85` +- Checked scripts: `86` +- Passed scripts: `86` - Failed scripts: `none` ## Script Surface @@ -81,6 +81,7 @@ | scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/html_rendering.py | internal-module | True | False | False | False | False | False | False | Used by report renderers to escape HTML while preserving meaningful falsey values. | +| scripts/import_output_review_decisions.py | cli | True | True | True | False | False | True | False | Imports human blind A/B reviewer decisions into the canonical output-review decision file. | | scripts/import_telemetry_events.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/init_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/judge_blind_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 4b4d72d2..f2993778 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -8,7 +8,7 @@ Scope: Yao Meta Skill against the user-provided Skill OS 2.0 upgrade plan. Yao Meta Skill is no longer only a Meta Skill factory. The current working tree now has the first verifiable Skill OS foundation: - Skill IR v0 for platform-neutral contract capture. -- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, and reviewer adjudication reporting. +- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, reviewer decision import, and adjudication reporting. - Runtime Conformance v0 for target-consumption checks. - Trust/Security v0 for secret, dependency, script, bounded network host policy, execution-level CLI help smoke checks, trust metadata, and stable source-contract integrity checks. - Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, aggregate drift signals, and no-route opportunities. @@ -40,6 +40,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Review Studio now avoids over-claiming release readiness when blind A/B adjudication, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, three warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion. - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. - Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. +- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, rejects raw prompt/output/transcript/message and answer-key fields, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. - Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes. @@ -51,7 +52,7 @@ This is still not the final world-class state. Target-native behavior contracts | --- | --- | --- | | Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed | | Target Compiler | `scripts/compile_skill.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | -| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | +| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | | Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed | | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | @@ -87,7 +88,7 @@ Next move: deepen provider-native execution transforms so OpenAI, Claude, VS Cod ### 2. Output eval now has command-runner evidence, but provider model execution is still open -The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, and an adjudication report from reviewer choices. The current root execution report now runs all `10` variants through `scripts/local_output_eval_runner.py`, so timing and command-runner behavior are observed instead of relying only on static recorded fixtures. The new `scripts/provider_output_eval_runner.py` gives teams a reviewed OpenAI Responses API compatible path for real model execution. The root evidence still intentionally reports `0` provider-backed model-executed runs and `10` estimated token counts, because no real provider holdout run has been recorded in this release evidence. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. The next gap is running provider-backed holdout cases with real provider timing/tokens and recording real reviewer decision records. +The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, an importer for reviewer JSON/JSONL/CSV decision sources, and an adjudication report from reviewer choices. The current root execution report now runs all `10` variants through `scripts/local_output_eval_runner.py`, so timing and command-runner behavior are observed instead of relying only on static recorded fixtures. The new `scripts/provider_output_eval_runner.py` gives teams a reviewed OpenAI Responses API compatible path for real model execution. The root evidence still intentionally reports `0` provider-backed model-executed runs and `10` estimated token counts, because no real provider holdout run has been recorded in this release evidence. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. The next gap is running provider-backed holdout cases with real provider timing/tokens and recording real reviewer decision records through the importer. Next move: run provider-backed holdout cases with real credentials and one real multi-file fixture, then record actual reviewer decisions. @@ -116,13 +117,13 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `33` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `85 / 85` CLI help smoke checks passing across `118` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `33` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `119` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `640` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `640` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `641` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `641` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `1` adoption sample, adoption `100`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 63e7fa9d..5c408585 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -83,6 +83,7 @@ "scripts/github_benchmark_scan.py", "scripts/governance_check.py", "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index d96b4d6d..fd313adc 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 118; help failures 0", + "current": "secrets 0; scripts 119; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 640; install failures 0; permission failures 0", + "current": "zip entries 641; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index e91fd44a..ca9afbd4 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 118; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 119; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 640; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 641; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 94ee2a0c..22841e62 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "118 scripts; secrets 0; help failures 0", + "current": "119 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -204,7 +204,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 640; install failures 0", + "current": "archive entries 641; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 39f0d6e4..b232831c 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,9 +26,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 118 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 119 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 640; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 641; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index cd3920bc..6f18370a 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "7d597e2b1385e312d344995cb934549731a7378d0f146d77a93f08d68101b9e4" + "to": "3b2b433b7c57d0a354a145f6755f84a1775b1429efa54f9b7a79ae809c9c79ae" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "dc875f356573a393c274e3dbee59dc85c78687add384e1e4a193957a57233183" + "to": "0037f80b78c16ae634875998f48b81b76734006923e2efbd9c40a35f8d5d9a75" } ] }, diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 396b0c9a..d42e2d05 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -183,7 +183,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -201,6 +202,7 @@ "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/human-adjudication.intake.json", "reports/world_class_evidence_intake.json", @@ -208,6 +210,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ] }, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index 33297ad1..bdc44eb2 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -121,12 +121,14 @@ This report validates the intake contract for human and external evidence. A val - reports/output_review_decisions.json - reports/output_review_adjudication.json - reports/output_review_adjudication.md + - scripts/import_output_review_decisions.py - evidence/world_class/intake.schema.json - evidence/world_class/templates/human-adjudication.intake.json - reports/world_class_evidence_intake.json - reports/world_class_evidence_intake.md - privacy_contract: - Reviewer decisions should not include raw user data or private customer detail. + - The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. - Keep the answer key separate until after decisions are recorded. #### Source Runbook @@ -134,7 +136,8 @@ This report validates the intake contract for human and external evidence. A val - `python3 scripts/yao.py output-review-kit --write-template` - Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. - `python3 scripts/adjudicate_output_review.py --write-template` -- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only. +- `python3 scripts/yao.py output-review-import --input --run-adjudication` - `python3 scripts/yao.py output-review` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index 7c395391..fc725830 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -176,7 +176,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -199,6 +200,7 @@ "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/human-adjudication.intake.json", "reports/world_class_evidence_intake.json", @@ -206,6 +208,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ], "observed_state": { diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 9f5a3944..7fd7dc45 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -91,7 +91,8 @@ This ledger records the current evidence state. It requires both passing source - `python3 scripts/yao.py output-review-kit --write-template` - Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. - `python3 scripts/adjudicate_output_review.py --write-template` -- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only. +- `python3 scripts/yao.py output-review-import --input --run-adjudication` - `python3 scripts/yao.py output-review` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. @@ -116,6 +117,7 @@ This ledger records the current evidence state. It requires both passing source ### Privacy Contract - Reviewer decisions should not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. - Keep the answer key separate until after decisions are recorded. ## Native Permission Enforcement diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index afaae4db..dbe36747 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -64,7 +64,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -82,6 +83,7 @@ "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/human-adjudication.intake.json", "reports/world_class_evidence_intake.json", @@ -89,6 +91,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ], "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." @@ -222,7 +225,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -240,6 +244,7 @@ "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/human-adjudication.intake.json", "reports/world_class_evidence_intake.json", @@ -247,6 +252,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ], "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index e97adc90..f2da2daa 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -68,7 +68,8 @@ This report is an execution plan for the remaining world-class evidence gaps. It - `python3 scripts/yao.py output-review-kit --write-template` - Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. - `python3 scripts/adjudicate_output_review.py --write-template` -- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only. +- `python3 scripts/yao.py output-review-import --input --run-adjudication` - `python3 scripts/yao.py output-review` - `python3 scripts/yao.py skill-os2-audit . --generated-at ` - Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. @@ -88,6 +89,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It - `reports/output_review_decisions.json` - `reports/output_review_adjudication.json` - `reports/output_review_adjudication.md` +- `scripts/import_output_review_decisions.py` - `evidence/world_class/intake.schema.json` - `evidence/world_class/templates/human-adjudication.intake.json` - `reports/world_class_evidence_intake.json` @@ -96,6 +98,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It ### Privacy Contract - Reviewer decisions should not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. - Keep the answer key separate until after decisions are recorded. ## Native Permission Enforcement diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index 6a0a8f10..086037e2 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -107,7 +107,7 @@
Action
Keep output holdout cases available before provider execution.
- +
file @@ -119,7 +119,7 @@
Action
Use the provider runner instead of the local command runner for model-backed evidence.
- +
env @@ -131,7 +131,7 @@
Action
Set OPENAI_API_KEY in the operator shell; never commit or print the value.
- +
env @@ -160,7 +160,7 @@
Action
Run provider-backed output-exec with real credentials.
- +
timing_observed_count @@ -173,7 +173,7 @@
Action
Provider execution should record timing metadata.
- +
token_observed_count @@ -193,7 +193,7 @@
  • YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
- +
human · ready-for-human-review @@ -226,7 +226,7 @@
Action
Open the blind review kit and record real reviewer choices.
- +
file @@ -235,10 +235,22 @@
Current
present
Status
pass
-
Action
Fill winner_variant values with real A/B decisions.
+
Action
Import real A/B decisions with `python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication`.
- + +
+
+ file + Decision importer +
+
+
Current
present
+
Status
pass
+
Action
Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.
+
+
+
human @@ -267,7 +279,7 @@
Action
Generate the blind A/B review pack.
- +
pending_count @@ -280,7 +292,7 @@
Action
Record a reviewer choice for every pair.
- +
judgment_count @@ -293,7 +305,7 @@
Action
Every pair needs one valid human judgment.
- +
invalid_decision_count @@ -310,10 +322,10 @@

Runbook

-
  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
- +
external · blocked @@ -346,7 +358,7 @@
Action
Keep approved high-permission capabilities explicit.
- +
file @@ -358,7 +370,7 @@
Action
Refresh runtime permission probes after packaging changes.
- +
external @@ -387,7 +399,7 @@
Action
Collect real target-client or external runtime guard proof.
- +
failure_count @@ -400,7 +412,7 @@
Action
Runtime permission probes must stay clean.
- +
installer_enforcement_ready @@ -420,7 +432,7 @@
  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
- +
external · blocked @@ -453,7 +465,7 @@
Action
Use the native host to receive metadata-only client events.
- +
file @@ -465,7 +477,7 @@
Action
Refresh telemetry hook recipes before external client installation.
- +
external @@ -494,7 +506,7 @@
Action
Import at least one metadata-only event from a real client.
- +
adoption_sample_count @@ -507,7 +519,7 @@
Action
Telemetry must include adoption outcome evidence.
- +
raw_content_allowed diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 74baff75..c957acb0 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -5,8 +5,8 @@ "skill_dir": ".", "summary": { "evidence_item_count": 4, - "precheck_count": 13, - "precheck_pass_count": 8, + "precheck_count": 14, + "precheck_pass_count": 9, "precheck_missing_count": 1, "precheck_optional_count": 1, "precheck_human_required_count": 1, @@ -27,7 +27,7 @@ "ready-for-human-review": 1 }, "precheck_status_counts": { - "pass": 8, + "pass": 9, "missing": 1, "optional": 1, "human-required": 1, @@ -184,12 +184,24 @@ "label": "Decision template", "kind": "file", "required": true, - "next_action": "Fill winner_variant values with real A/B decisions.", + "next_action": "Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`.", "secret_value_redacted": false, "path": "reports/output_review_decisions.json", "status": "pass", "actual": "present" }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, { "evidence_key": "human-adjudication", "key": "human-reviewer", @@ -268,7 +280,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -564,12 +577,24 @@ "label": "Decision template", "kind": "file", "required": true, - "next_action": "Fill winner_variant values with real A/B decisions.", + "next_action": "Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`.", "secret_value_redacted": false, "path": "reports/output_review_decisions.json", "status": "pass", "actual": "present" }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, { "evidence_key": "human-adjudication", "key": "human-reviewer", diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index ea962bc7..d14e8968 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -75,7 +75,8 @@ Generate the submission kit after the real provider, human, native-permission, o | Check | Kind | Current | Status | Next action | | --- | --- | --- | --- | --- | | Blind review kit | `file` | `present` | `pass` | Open the blind review kit and record real reviewer choices. | -| Decision template | `file` | `present` | `pass` | Fill winner_variant values with real A/B decisions. | +| Decision template | `file` | `present` | `pass` | Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`. | +| Decision importer | `file` | `present` | `pass` | Use the importer to reject raw content fields and normalize reviewer decisions before adjudication. | | Human reviewer | `human` | `external-human-action` | `human-required` | Assign a real reviewer identity before claiming human adjudication. | ### Source Checks diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index be9f2679..81f44874 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -87,12 +87,12 @@
Blocked
2
Submission
evidence/world_class/submissions/human-adjudication.json
-

Source Runbook

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Source Runbook

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .

Must Collect

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

Success Checks

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
-

Privacy

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
+

Privacy

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.

Next Source Actions

  • Record a reviewer choice for every pair.
  • Every pair needs one valid human judgment.

Source Evidence Snapshot

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index 1ad88b47..c889b801 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -149,7 +149,8 @@ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", @@ -179,6 +180,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ] }, @@ -188,6 +190,7 @@ "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/human-adjudication.intake.json", "reports/world_class_evidence_intake.json", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 715b410a..7d396272 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -107,7 +107,8 @@ This runbook coordinates evidence collection only. It does not accept submission - python3 scripts/yao.py output-review-kit --write-template - Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. - python3 scripts/adjudicate_output_review.py --write-template -- Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata. +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only. +- python3 scripts/yao.py output-review-import --input --run-adjudication - python3 scripts/yao.py output-review - python3 scripts/yao.py skill-os2-audit . --generated-at - Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. @@ -137,6 +138,7 @@ This runbook coordinates evidence collection only. It does not accept submission ### Privacy Contract - Reviewer decisions should not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. - Keep the answer key separate until after decisions are recorded. ### Evidence Artifacts @@ -146,6 +148,7 @@ This runbook coordinates evidence collection only. It does not accept submission - reports/output_review_decisions.json - reports/output_review_adjudication.json - reports/output_review_adjudication.md +- scripts/import_output_review_decisions.py - evidence/world_class/intake.schema.json - evidence/world_class/templates/human-adjudication.intake.json - reports/world_class_evidence_intake.json diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 867863f1..43a75d25 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -206,6 +206,7 @@ ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded." ], "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 899b7aa1..7d625fcc 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -91,6 +91,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Privacy Contract - Reviewer decisions should not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. - Keep the answer key separate until after decisions are recorded. ### Native Permission Enforcement diff --git a/scripts/adjudicate_output_review.py b/scripts/adjudicate_output_review.py index 6974f0c8..76dcf1f5 100644 --- a/scripts/adjudicate_output_review.py +++ b/scripts/adjudicate_output_review.py @@ -247,6 +247,7 @@ def build_reviewer_checklist( "commands": { "prepare_review_kit": "python3 scripts/yao.py output-review-kit", "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --run-adjudication", "adjudicate": "python3 scripts/yao.py output-review", "refresh_review_studio": "python3 scripts/yao.py review-studio .", }, diff --git a/scripts/import_output_review_decisions.py b/scripts/import_output_review_decisions.py new file mode 100644 index 00000000..2ab69773 --- /dev/null +++ b/scripts/import_output_review_decisions.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +import argparse +import csv +import json +from datetime import date +from pathlib import Path +from typing import Any + +from adjudicate_output_review import ( + DEFAULT_ANSWER_KEY, + DEFAULT_BLIND_PACK, + DEFAULT_DECISIONS, + DEFAULT_OUTPUT_JSON, + DEFAULT_OUTPUT_MD, + adjudicate_output_review, + build_decision_template, + normalize_variant, + pair_index, +) + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file." + +RAW_CONTENT_FIELDS = { + "prompt", + "prompts", + "input", + "inputs", + "output", + "outputs", + "transcript", + "transcripts", + "message", + "messages", + "user_message", + "assistant_message", + "model_output", + "baseline_output", + "with_skill_output", + "raw_content", + "raw_prompt", + "raw_output", +} + +ANSWER_KEY_FIELDS = { + "expected", + "expected_winner", + "expected_winner_variant", + "answer_key", + "label", + "variant_label", + "baseline_label", + "with_skill_label", +} + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> tuple[Any, list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except json.JSONDecodeError as exc: + return None, [f"Invalid JSON file {display_path(path)}: {exc}"] + + +def detect_format(path: Path, requested: str) -> str: + if requested != "auto": + return requested + suffix = path.suffix.lower() + if suffix == ".jsonl": + return "jsonl" + if suffix == ".csv": + return "csv" + return "json" + + +def read_decision_source(path: Path, source_format: str) -> tuple[dict[str, Any], list[dict[str, Any]], list[str]]: + if not path.exists(): + return {}, [], [f"Missing decision source: {display_path(path)}"] + if source_format == "json": + payload, failures = load_json(path) + if failures: + return {}, [], failures + if isinstance(payload, list): + return {}, payload, [] + if not isinstance(payload, dict): + return {}, [], ["JSON decision source must be an object or list"] + decisions = payload.get("decisions", []) + if not isinstance(decisions, list): + return payload, [], ["decisions must be a list"] + return payload, decisions, [] + if source_format == "jsonl": + decisions: list[dict[str, Any]] = [] + failures: list[str] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + failures.append(f"JSONL line {line_number} is invalid: {exc}") + continue + if not isinstance(item, dict): + failures.append(f"JSONL line {line_number} must be an object") + continue + decisions.append(item) + return {}, decisions, failures + if source_format == "csv": + with path.open("r", encoding="utf-8", newline="") as handle: + return {}, [dict(row) for row in csv.DictReader(handle)], [] + return {}, [], [f"Unsupported decision source format: {source_format}"] + + +def known_case_ids(blind_pack_path: Path) -> tuple[set[str], list[str]]: + payload, failures = load_json(blind_pack_path) + if failures: + return set(), failures + if not isinstance(payload, dict): + return set(), [f"Blind pack root must be an object: {display_path(blind_pack_path)}"] + pairs = pair_index(payload) + if not pairs: + return set(), [f"Blind pack has no pairs: {display_path(blind_pack_path)}"] + return set(pairs), [] + + +def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None]: + if value is None or str(value).strip() == "": + return None, None + try: + parsed = float(value) + except (TypeError, ValueError): + return None, f"{case_id}: confidence must be numeric" + if parsed < 0 or parsed > 1: + return None, f"{case_id}: confidence must be between 0 and 1" + return round(parsed, 3), None + + +def forbidden_fields(item: dict[str, Any]) -> list[str]: + keys = {str(key) for key in item} + return sorted((keys & RAW_CONTENT_FIELDS) | (keys & ANSWER_KEY_FIELDS)) + + +def normalize_decisions( + source_items: list[dict[str, Any]], + case_ids: set[str], +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + normalized: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, item in enumerate(source_items, start=1): + if not isinstance(item, dict): + failures.append(f"decision #{index} must be an object") + continue + blocked_fields = forbidden_fields(item) + if blocked_fields: + failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}") + case_id = str(item.get("case_id", "")).strip() + if not case_id: + failures.append(f"decision #{index} is missing case_id") + continue + if case_id not in case_ids: + failures.append(f"decision #{index} references unknown case_id: {case_id}") + if case_id in seen: + failures.append(f"duplicate decision for case_id: {case_id}") + seen.add(case_id) + winner = normalize_variant(item.get("winner_variant", item.get("winner", item.get("reviewer_winner_variant", "")))) + if winner and winner not in {"A", "B"}: + failures.append(f"{case_id}: winner_variant must be A or B") + confidence, confidence_failure = parse_confidence(item.get("confidence"), case_id) + if confidence_failure: + failures.append(confidence_failure) + normalized.append( + { + "case_id": case_id, + "winner_variant": winner, + "confidence": confidence, + "reason": str(item.get("reason", "")).strip(), + } + ) + return normalized, failures + + +def canonical_payload( + blind_pack_path: Path, + source_path: Path, + source_format: str, + reviewer: str, + reviewed_at: str, + decisions: list[dict[str, Any]], +) -> dict[str, Any]: + template = build_decision_template(json.loads(blind_pack_path.read_text(encoding="utf-8"))) + completed = sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"}) + return { + "schema_version": "1.0", + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "decision_contract": template["decision_contract"], + "import_contract": { + "schema_version": "1.0", + "source_path": display_path(source_path), + "source_format": source_format, + "raw_content_allowed": False, + "answer_key_fields_allowed": False, + "answer_key_opened_by_importer": False, + "all_or_nothing_write": True, + "decision_count": len(decisions), + "completed_decision_count": completed, + "pending_decision_count": len(decisions) - completed, + }, + "decisions": decisions, + } + + +def import_output_review_decisions( + source_path: Path, + blind_pack_path: Path, + output_json: Path, + source_format: str = "auto", + reviewer: str = "", + reviewed_at: str = "", + run_adjudication: bool = False, + answer_key_path: Path = DEFAULT_ANSWER_KEY, + adjudication_json: Path = DEFAULT_OUTPUT_JSON, + adjudication_md: Path = DEFAULT_OUTPUT_MD, +) -> dict[str, Any]: + detected_format = detect_format(source_path, source_format) + source_meta, source_items, failures = read_decision_source(source_path, detected_format) + reviewer = reviewer or str(source_meta.get("reviewer", "")).strip() + reviewed_at = reviewed_at or str(source_meta.get("reviewed_at", "")).strip() + if not reviewer: + failures.append("reviewer is required for imported human decisions") + if not reviewed_at: + failures.append("reviewed_at is required for imported human decisions") + case_ids, case_failures = known_case_ids(blind_pack_path) + failures.extend(case_failures) + decisions, decision_failures = normalize_decisions(source_items, case_ids) + failures.extend(decision_failures) + payload: dict[str, Any] = { + "schema_version": "1.0", + "ok": not failures, + "summary": { + "decision_count": len(decisions), + "completed_decision_count": sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"}), + "pending_decision_count": sum(1 for item in decisions if not item.get("winner_variant")), + "failure_count": len(failures), + "canonical_written": False, + "adjudication_run": False, + }, + "artifacts": { + "source": display_path(source_path), + "blind_pack": display_path(blind_pack_path), + "decisions": display_path(output_json), + "adjudication_json": display_path(adjudication_json), + "adjudication_markdown": display_path(adjudication_md), + }, + "failures": failures, + } + if failures: + return payload + canonical = canonical_payload(blind_pack_path, source_path, detected_format, reviewer, reviewed_at, decisions) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(canonical, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + payload["summary"]["canonical_written"] = True + payload["canonical"] = canonical + if run_adjudication: + adjudication = adjudicate_output_review( + blind_pack_path=blind_pack_path, + answer_key_path=answer_key_path, + decisions_path=output_json, + output_json=adjudication_json, + output_md=adjudication_md, + ) + payload["summary"]["adjudication_run"] = True + payload["summary"]["adjudication_ok"] = adjudication["ok"] + payload["summary"]["adjudication_pending_count"] = adjudication["summary"]["pending_count"] + payload["summary"]["adjudication_invalid_decision_count"] = adjudication["summary"]["invalid_decision_count"] + payload["adjudication_summary"] = adjudication["summary"] + if not adjudication["ok"]: + payload["ok"] = False + payload["failures"] = list(adjudication.get("failures", [])) + payload["summary"]["failure_count"] = len(payload["failures"]) + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Import blind A/B reviewer decisions into the canonical output-review decision file.") + parser.add_argument("--input", required=True, help="Reviewer decision source in JSON, JSONL, or CSV format.") + parser.add_argument("--format", choices=["auto", "json", "jsonl", "csv"], default="auto") + parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK)) + parser.add_argument("--output-json", default=str(DEFAULT_DECISIONS)) + parser.add_argument("--reviewer") + parser.add_argument("--reviewed-at", default=date.today().isoformat()) + parser.add_argument("--run-adjudication", action="store_true") + parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY)) + parser.add_argument("--adjudication-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--adjudication-md", default=str(DEFAULT_OUTPUT_MD)) + args = parser.parse_args() + payload = import_output_review_decisions( + source_path=Path(args.input).resolve(), + blind_pack_path=Path(args.blind_pack).resolve(), + output_json=Path(args.output_json).resolve(), + source_format=args.format, + reviewer=args.reviewer or "", + reviewed_at=args.reviewed_at or "", + run_adjudication=args.run_adjudication, + answer_key_path=Path(args.answer_key).resolve(), + adjudication_json=Path(args.adjudication_json).resolve(), + adjudication_md=Path(args.adjudication_md).resolve(), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py index 48d339c7..eb0b7c47 100644 --- a/scripts/render_world_class_evidence_plan.py +++ b/scripts/render_world_class_evidence_plan.py @@ -44,7 +44,8 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", "python3 scripts/adjudicate_output_review.py --write-template", - "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", "python3 scripts/yao.py output-review", "python3 scripts/yao.py skill-os2-audit . --generated-at ", ], @@ -60,9 +61,11 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", ], "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", "Keep the answer key separate until after decisions are recorded.", ], }, diff --git a/scripts/render_world_class_preflight.py b/scripts/render_world_class_preflight.py index 50712501..a3c3eafd 100644 --- a/scripts/render_world_class_preflight.py +++ b/scripts/render_world_class_preflight.py @@ -72,7 +72,15 @@ PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = { "kind": "file", "path": "reports/output_review_decisions.json", "required": True, - "next_action": "Fill winner_variant values with real A/B decisions.", + "next_action": "Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`.", + }, + { + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "path": "scripts/import_output_review_decisions.py", + "required": True, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", }, { "key": "human-reviewer", @@ -562,7 +570,7 @@ def render_html(report: dict[str, Any]) -> str: for label, value in stats ) item_cards = "".join(render_html_item(item) for item in report.get("items", [])) - return f""" + html = f""" @@ -647,6 +655,7 @@ def render_html(report: dict[str, Any]) -> str: """ + return "\n".join(line.rstrip() for line in html.splitlines()) + "\n" def main() -> None: diff --git a/scripts/yao.py b/scripts/yao.py index f909d4c7..6d99dd06 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -35,6 +35,7 @@ from yao_cli_output_commands import ( command_output_eval, command_output_execution, command_output_review, + command_output_review_import, command_output_review_kit, ) from yao_cli_parser import build_parser as build_cli_parser diff --git a/scripts/yao_cli_output_commands.py b/scripts/yao_cli_output_commands.py index 403119cd..1d1c4a5d 100644 --- a/scripts/yao_cli_output_commands.py +++ b/scripts/yao_cli_output_commands.py @@ -104,3 +104,26 @@ def command_output_review(args: argparse.Namespace) -> int: if args.write_template: cmd.append("--write-template") return emit_result(run_script("adjudicate_output_review.py", cmd)) + + +def command_output_review_import(args: argparse.Namespace) -> int: + cmd = ["--input", args.input] + if args.format: + cmd.extend(["--format", args.format]) + if args.blind_pack: + cmd.extend(["--blind-pack", args.blind_pack]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.reviewer: + cmd.extend(["--reviewer", args.reviewer]) + if args.reviewed_at: + cmd.extend(["--reviewed-at", args.reviewed_at]) + if args.run_adjudication: + cmd.append("--run-adjudication") + if args.answer_key: + cmd.extend(["--answer-key", args.answer_key]) + if args.adjudication_json: + cmd.extend(["--adjudication-json", args.adjudication_json]) + if args.adjudication_md: + cmd.extend(["--adjudication-md", args.adjudication_md]) + return emit_result(run_script("import_output_review_decisions.py", cmd)) diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 430c542b..ccd2ba28 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -759,6 +759,22 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] output_review_cmd.add_argument("--write-template", action="store_true") output_review_cmd.set_defaults(func=_handler(command_handlers, "command_output_review")) + output_review_import_cmd = subparsers.add_parser( + "output-review-import", + help="Import human blind A/B reviewer decisions into the canonical decision file.", + ) + output_review_import_cmd.add_argument("--input", required=True) + output_review_import_cmd.add_argument("--format", choices=["auto", "json", "jsonl", "csv"], default="auto") + output_review_import_cmd.add_argument("--blind-pack") + output_review_import_cmd.add_argument("--output-json") + output_review_import_cmd.add_argument("--reviewer") + output_review_import_cmd.add_argument("--reviewed-at") + output_review_import_cmd.add_argument("--run-adjudication", action="store_true") + output_review_import_cmd.add_argument("--answer-key") + output_review_import_cmd.add_argument("--adjudication-json") + output_review_import_cmd.add_argument("--adjudication-md") + output_review_import_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_import")) + conformance_cmd = subparsers.add_parser("conformance", help="Run runtime conformance checks for Skill OS targets.") conformance_cmd.add_argument("skill_dir", nargs="?", default=".") conformance_cmd.add_argument("--target", action="append", choices=["openai", "claude", "agent-skills", "vscode", "generic"]) diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index 53d2636b..d6c55d86 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -52,10 +52,11 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_iteration_ledger.py", []), run_script("render_baseline_compare.py", baseline_compare_args()), run_script("render_regression_history.py", []), - run_script("render_context_reports.py", []), - run_script("render_portability_report.py", []), + run_script("trust_check.py", [str(ROOT)]), run_script("python_compat_check.py", [str(ROOT)]), run_script("render_architecture_maintainability.py", [str(ROOT)]), + run_script("render_context_reports.py", []), + run_script("render_portability_report.py", []), run_script("render_reference_synthesis.py", [str(ROOT)]), run_script("render_artifact_design_profile.py", [str(ROOT)]), run_script("render_prompt_quality_profile.py", [str(ROOT)]), @@ -100,6 +101,7 @@ def command_report(args: argparse.Namespace) -> int: "regression_history": "reports/regression_history.md", "context_budget": "reports/context_budget.json", "portability_score": "reports/portability_score.json", + "security_trust": "reports/security_trust_report.json", "python_compatibility": "reports/python_compatibility.json", "architecture_maintainability": "reports/architecture_maintainability.json", "reference_synthesis": "reports/reference-synthesis.json", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index de2a16eb..18a28f75 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -39,6 +39,7 @@ "scripts/github_benchmark_scan.py", "scripts/governance_check.py", "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py index a69d1478..b9713dcc 100644 --- a/tests/verify_output_review_adjudication.py +++ b/tests/verify_output_review_adjudication.py @@ -9,6 +9,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" ADJUDICATOR = ROOT / "scripts" / "adjudicate_output_review.py" +IMPORTER = ROOT / "scripts" / "import_output_review_decisions.py" +CLI = ROOT / "scripts" / "yao.py" def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: @@ -87,6 +89,7 @@ def main() -> None: assert checklist["skill-package-contract"]["answer_key_visible"] is False, checklist["skill-package-contract"] assert checklist["skill-package-contract"]["decisions_path"].endswith("tests/tmp_output_review_adjudication/missing_decisions.json"), checklist["skill-package-contract"] assert checklist["skill-package-contract"]["commands"]["write_template"] == "python3 scripts/adjudicate_output_review.py --write-template", checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["commands"]["import_decisions"].startswith("python3 scripts/yao.py output-review-import"), checklist["skill-package-contract"] assert checklist["skill-package-contract"]["required_fields"]["winner_variant"].startswith("A or B"), checklist["skill-package-contract"] assert "No reviewer decisions recorded yet" in pending_md.read_text(encoding="utf-8"), pending_md pending_text = pending_md.read_text(encoding="utf-8") @@ -167,6 +170,180 @@ def main() -> None: filled_checklist = {item["case_id"]: item for item in filled_payload["reviewer_checklist"]} assert all(item["readiness"] == "adjudicated" and item["answer_key_visible"] for item in filled_checklist.values()), filled_checklist + import_source = tmp_root / "reviewer_source.json" + import_source.write_text(json.dumps(decisions, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + imported_path = tmp_root / "imported_decisions.json" + imported_proc = run( + [ + str(IMPORTER), + "--input", + str(import_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(imported_path), + "--run-adjudication", + "--answer-key", + str(answer_key_json), + "--adjudication-json", + str(tmp_root / "imported_adjudication.json"), + "--adjudication-md", + str(tmp_root / "imported_adjudication.md"), + ] + ) + imported_payload = json.loads(imported_proc.stdout) + assert imported_payload["ok"], imported_payload + assert imported_payload["summary"]["canonical_written"] is True, imported_payload + assert imported_payload["summary"]["adjudication_run"] is True, imported_payload + assert imported_payload["summary"]["adjudication_pending_count"] == 0, imported_payload + imported_decisions = json.loads(imported_path.read_text(encoding="utf-8")) + assert imported_decisions["import_contract"]["raw_content_allowed"] is False, imported_decisions + assert imported_decisions["import_contract"]["answer_key_fields_allowed"] is False, imported_decisions + assert imported_decisions["import_contract"]["answer_key_opened_by_importer"] is False, imported_decisions + assert imported_decisions["reviewer"] == "Yao QA", imported_decisions + + cli_import_proc = run( + [ + str(CLI), + "output-review-import", + "--input", + str(import_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "cli_imported_decisions.json"), + "--run-adjudication", + "--answer-key", + str(answer_key_json), + "--adjudication-json", + str(tmp_root / "cli_imported_adjudication.json"), + "--adjudication-md", + str(tmp_root / "cli_imported_adjudication.md"), + ] + ) + cli_import_payload = json.loads(cli_import_proc.stdout) + assert cli_import_payload["ok"], cli_import_payload + assert cli_import_payload["summary"]["canonical_written"] is True, cli_import_payload + assert cli_import_payload["summary"]["adjudication_pending_count"] == 0, cli_import_payload + + jsonl_source = tmp_root / "reviewer_source.jsonl" + jsonl_source.write_text("\n".join(json.dumps(item, ensure_ascii=False) for item in decisions["decisions"]) + "\n", encoding="utf-8") + jsonl_proc = run( + [ + str(IMPORTER), + "--input", + str(jsonl_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "jsonl_decisions.json"), + "--reviewer", + "Yao QA", + "--reviewed-at", + "2026-06-13", + ] + ) + jsonl_payload = json.loads(jsonl_proc.stdout) + assert jsonl_payload["ok"], jsonl_payload + assert jsonl_payload["summary"]["completed_decision_count"] == 5, jsonl_payload + + csv_source = tmp_root / "reviewer_source.csv" + csv_source.write_text( + "case_id,winner_variant,confidence,reason\n" + + "\n".join( + f"{item['case_id']},{item['expected_winner_variant']},0.8,CSV reviewer choice" + for item in answer_key["answers"] + ) + + "\n", + encoding="utf-8", + ) + csv_proc = run( + [ + str(IMPORTER), + "--input", + str(csv_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "csv_decisions.json"), + "--reviewer", + "Yao QA", + "--reviewed-at", + "2026-06-13", + ] + ) + csv_payload = json.loads(csv_proc.stdout) + assert csv_payload["ok"], csv_payload + assert csv_payload["summary"]["completed_decision_count"] == 5, csv_payload + + private_source = tmp_root / "private_source.json" + private_source.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "A", + "prompt": "raw prompt must not be imported", + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + private_proc = run( + [ + str(IMPORTER), + "--input", + str(private_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "private_decisions.json"), + ], + check=False, + ) + private_payload = json.loads(private_proc.stdout) + assert private_proc.returncode == 2, private_payload + assert private_payload["ok"] is False, private_payload + assert any("forbidden raw or answer-key fields" in failure for failure in private_payload["failures"]), private_payload + assert not (tmp_root / "private_decisions.json").exists(), private_payload + + unknown_case = tmp_root / "unknown_case.json" + unknown_case.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [{"case_id": "unknown", "winner_variant": "A"}], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + unknown_proc = run( + [ + str(IMPORTER), + "--input", + str(unknown_case), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "unknown_decisions.json"), + ], + check=False, + ) + unknown_payload = json.loads(unknown_proc.stdout) + assert unknown_proc.returncode == 2, unknown_payload + assert any("unknown case_id" in failure for failure in unknown_payload["failures"]), unknown_payload + invalid = { "schema_version": "1.0", "reviewer": "Yao QA", diff --git a/tests/verify_world_class_evidence_plan.py b/tests/verify_world_class_evidence_plan.py index 1e16d69a..0a72b6d0 100644 --- a/tests/verify_world_class_evidence_plan.py +++ b/tests/verify_world_class_evidence_plan.py @@ -57,7 +57,10 @@ def main() -> None: assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("world-class-intake . --submissions-dir evidence/world_class/submissions" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] - assert any("output_review_decisions.json" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"] + assert "reports/output_review_decisions.json" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"] + assert any("output-review-import" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"] + assert "scripts/import_output_review_decisions.py" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"] + assert any("importer rejects raw prompt" in item for item in tasks["human-adjudication"]["privacy_contract"]), tasks["human-adjudication"] assert any("runtime-permissions" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"] assert any("install-simulate" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"] assert any("summary.failure_count == 0" in check for check in tasks["native-permission-enforcement"]["success_checks"]), tasks["native-permission-enforcement"] diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 20e460a2..c0e4095d 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -80,7 +80,7 @@ def main() -> None: expected_help = ( "quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " "world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard " - "benchmark-reproducibility evidence-consistency output-review-kit adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry" + "benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry" ).split() assert all(item in parser_help for item in expected_help), parser_help