diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 43d1a3cb..1b14ed13 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 209, - "script_file_count": 136, - "test_file_count": 73, - "internal_module_count": 52, + "python_file_count": 211, + "script_file_count": 137, + "test_file_count": 74, + "internal_module_count": 53, "cli_script_count": 86, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -15,27 +15,13 @@ "warn_line_threshold": 900, "watch_line_threshold": 720, "block_line_threshold": 1500, - "largest_file_lines": 704, + "largest_file_lines": 696, "watchlist_count": 0, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" }, "largest_files": [ - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 704, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, - { - "path": "scripts/world_class_evidence_contract.py", - "lines": 699, - "kind": "internal-module", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "tests/verify_yao_cli.py", "lines": 696, @@ -99,12 +85,26 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/world_class_evidence_contract.py", + "lines": 620, + "kind": "internal-module", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/render_benchmark_reproducibility.py", "lines": 595, "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/optimize_description.py", + "lines": 585, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } ], "watchlist": [], diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 026a9923..35a675de 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,15 +5,15 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `209` -- scripts: `136` -- tests: `73` -- internal modules: `52` +- python files: `211` +- scripts: `137` +- tests: `74` +- internal modules: `53` - CLI scripts: `86` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` - command modules: `6` -- largest file lines: `704` +- largest file lines: `696` - watch threshold lines: `720` - watchlist: `0` - hotspots: `0` @@ -33,8 +33,6 @@ No near-threshold files found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | -| `tests/verify_world_class_evidence_intake.py` | `704` | `test` | `pass` | -| `scripts/world_class_evidence_contract.py` | `699` | `internal-module` | `pass` | | `tests/verify_yao_cli.py` | `696` | `test` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` | @@ -44,7 +42,9 @@ No near-threshold files found. | `scripts/review_studio_gates.py` | `643` | `internal-module` | `pass` | | `scripts/cross_packager.py` | `638` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `637` | `cli-script` | `pass` | +| `scripts/world_class_evidence_contract.py` | `620` | `internal-module` | `pass` | | `scripts/render_benchmark_reproducibility.py` | `595` | `cli-script` | `pass` | +| `scripts/optimize_description.py` | `585` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/context_budget.json b/reports/context_budget.json index eeb0b581..8f66ae90 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1060977, + "other_text_tokens": 1063452, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1061774, - "deferred_resource_tokens": 491281, + "estimated_total_text_tokens": 1064249, + "deferred_resource_tokens": 492676, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 431183, - "file_count": 136 + "estimated_tokens": 432578, + "file_count": 137 }, { "path": "references", @@ -36,8 +36,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 431183, - "file_count": 136 + "estimated_tokens": 432578, + "file_count": 137 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 431183, - "file_count": 136, + "estimated_tokens": 432578, + "file_count": 137, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 665, + "relevant_file_count": 669, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index 69db51a6..03154374 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 491281 | `governed` | scripts:431183 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 492676 | `governed` | scripts:432578 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index b6d99af6..23202e9d 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 491281, + "deferred_resource_tokens": 492676, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 431183, - "file_count": 136 + "estimated_tokens": 432578, + "file_count": 137 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 431183, - "file_count": 136, + "estimated_tokens": 432578, + "file_count": 137, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index fec57cbd..75e852a9 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 491281/120000; top deferred scripts 431183; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 492676/120000; top deferred scripts 432578; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 491281/120000; top deferred scripts 431183; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 492676/120000; top deferred scripts 432578; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -2404,8 +2404,8 @@ "score `91`", "`16` gates", "`3` warnings", - "`50` declared internal modules", - "`86 / 86` CLI help smoke checks passing across `136` scripts", + "`51` declared internal modules", + "`86 / 86` CLI help smoke checks passing across `137` scripts", "`658` zip entries", "archive with `658` entries", "`12` installer permission checks enforced", diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 4b834f26..f5e37ebb 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 212, + "file_count": 214, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -754,6 +754,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_human_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/world_class_preflight_layout.py", "ok": true, @@ -1282,6 +1288,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/world_class_human_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/yao_cli_helpers.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 357eb04a..9d1c8aa6 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `212` +- files scanned: `214` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 0719babd..58ab11a6 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 227, - "script_count": 136, - "internal_module_count": 50, + "scanned_files": 228, + "script_count": 137, + "internal_module_count": 51, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 227, - "package_sha256": "01c09d5fa7c971a657e4d138499ac8d50095ae1a078c2bff1bd9b70657199c9f" + "package_hash_file_count": 228, + "package_sha256": "c2ad60de7b1e3d70021ec9e0169a8f8c8c860ff830a434baba2ce44e2530069e" }, "failures": [], "warnings": [], @@ -1712,6 +1712,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_human_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/world_class_preflight_layout.py", "interface": "internal-module", @@ -1985,7 +1999,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 50, + "skipped_count": 51, "failed_scripts": [], "results": [ { @@ -2986,6 +3000,10 @@ "path": "scripts/world_class_evidence_contract.py", "reason": "internal module" }, + { + "path": "scripts/world_class_human_evidence.py", + "reason": "internal module" + }, { "path": "scripts/world_class_preflight_layout.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 06caf9ec..493da8c3 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `227` -- Scripts: `136` -- Internal script modules: `50` +- Scanned files: `228` +- Scripts: `137` +- Internal script modules: `51` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `227` -- Package SHA256: `01c09d5fa7c971a657e4d138499ac8d50095ae1a078c2bff1bd9b70657199c9f` +- Package hash files: `228` +- Package SHA256: `c2ad60de7b1e3d70021ec9e0169a8f8c8c860ff830a434baba2ce44e2530069e` ## Failures @@ -181,6 +181,7 @@ | scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | +| scripts/world_class_human_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows. | | scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. | | scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. | | scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 9d762b88..82887088 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -128,7 +128,7 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `50` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `136` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `51` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `137` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 6b6492d3..b4023720 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 136; help failures 0", + "current": "secrets 0; scripts 137; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 8aab454c..bb985cf5 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 136; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 137; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index eb8b70b2..59682130 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "136 scripts; secrets 0; help failures 0", + "current": "137 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 38d45db4..a36caa8a 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 136 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 137 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 658; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index d32a4fd4..f9c184de 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from world_class_human_evidence import validate_human_adjudication_report from world_class_provider_evidence import validate_provider_execution_report @@ -367,88 +368,8 @@ def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[ decisions = load_json(paths.get("reports/output_review_decisions.json", root / "__missing__")) if not adjudication or not decisions: return - - summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {} - pair_count = real_int(summary.get("pair_count")) - judgment_count = real_int(summary.get("judgment_count")) - pending_count = real_int(summary.get("pending_count")) - invalid_decision_count = real_int(summary.get("invalid_decision_count")) - answer_revealed_count = real_int(summary.get("answer_revealed_count")) - pending_answer_hidden_count = real_int(summary.get("pending_answer_hidden_count")) - add_error(errors, bool(pair_count and pair_count > 0), "human-adjudication adjudication summary.pair_count must be >0") - add_error( - errors, - bool(pair_count and judgment_count == pair_count), - "human-adjudication adjudication summary.judgment_count must equal summary.pair_count", - ) - add_error(errors, pending_count == 0, "human-adjudication adjudication summary.pending_count must be 0") - add_error( - errors, - invalid_decision_count == 0, - "human-adjudication adjudication summary.invalid_decision_count must be 0", - ) - add_error( - errors, - bool(pair_count and answer_revealed_count == pair_count), - "human-adjudication adjudication summary.answer_revealed_count must equal summary.pair_count", - ) - add_error( - errors, - pending_answer_hidden_count == 0, - "human-adjudication adjudication summary.pending_answer_hidden_count must be 0", - ) - add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false") - - decision_rows = decisions.get("decisions", []) - add_error(errors, decisions.get("schema_version") == "1.0", "human-adjudication decisions.schema_version must be 1.0") - reviewer = str(decisions.get("reviewer", "")).strip() - reviewed_at = str(decisions.get("reviewed_at", "")).strip() - require_real_text(errors, reviewer, "human-adjudication decisions.reviewer") - add_error( - errors, - bool(SUBMITTED_AT_RE.match(reviewed_at)), - "human-adjudication decisions.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - ) - add_error( - errors, - isinstance(decision_rows, list) and bool(pair_count and len(decision_rows) == pair_count), - "human-adjudication decisions count must equal adjudication summary.pair_count", - ) - if isinstance(decision_rows, list): - invalid_winners = [ - str(item.get("case_id", index + 1)) - for index, item in enumerate(decision_rows) - if not isinstance(item, dict) or str(item.get("winner_variant", "")).strip().upper() not in {"A", "B"} - ] - missing_reasons = [ - str(item.get("case_id", index + 1)) - for index, item in enumerate(decision_rows) - if not isinstance(item, dict) or not str(item.get("reason", "")).strip() - ] - add_error( - errors, - not invalid_winners, - "human-adjudication decisions must include A/B winner_variant for every case", - ) - add_error(errors, not missing_reasons, "human-adjudication decisions must include reviewer reason for every case") - provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} - provenance_reviewer = str(provenance.get("reviewer", "")).strip() - add_error( - errors, - bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer), - "human-adjudication provenance.reviewer must match decisions.reviewer", - ) - add_error( - errors, - adjudication.get("reviewer") == reviewer, - "human-adjudication adjudication reviewer must match decisions.reviewer", - ) - add_error( - errors, - adjudication.get("reviewed_at") == reviewed_at, - "human-adjudication adjudication reviewed_at must match decisions.reviewed_at", - ) + validate_human_adjudication_report(adjudication, decisions, provenance, errors) def validate_native_permission_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: diff --git a/scripts/world_class_human_evidence.py b/scripts/world_class_human_evidence.py new file mode 100644 index 00000000..ec96136e --- /dev/null +++ b/scripts/world_class_human_evidence.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +import re +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows." + +REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$") +RAW_CONTENT_FIELDS = { + "assistant_message", + "assistant_messages", + "baseline_output", + "input", + "inputs", + "message", + "messages", + "model_output", + "output", + "outputs", + "prompt", + "prompts", + "raw_content", + "raw_output", + "raw_prompt", + "transcript", + "transcripts", + "user_message", + "user_messages", + "with_skill_output", +} +ANSWER_KEY_FIELDS = { + "answer_key", + "baseline_label", + "expected", + "expected_winner", + "expected_winner_role", + "expected_winner_variant", + "label", + "variant_label", + "with_skill_label", +} + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def require_real_text(errors: list[str], value: Any, field: str) -> None: + add_error(errors, bool(str(value or "").strip()), f"{field} is required") + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def case_ids(items: list[dict[str, Any]]) -> list[str]: + return [str(item.get("case_id", "")).strip() for item in items] + + +def duplicate_case_ids(ids: list[str]) -> list[str]: + seen: set[str] = set() + duplicates: list[str] = [] + for case_id in ids: + if case_id in seen and case_id not in duplicates: + duplicates.append(case_id) + seen.add(case_id) + return duplicates + + +def forbidden_decision_fields(item: dict[str, Any]) -> list[str]: + keys = {str(key).strip().lower() for key in item} + return sorted((keys & RAW_CONTENT_FIELDS) | (keys & ANSWER_KEY_FIELDS)) + + +def confidence_valid(value: Any) -> bool: + if value is None or value == "": + return True + if isinstance(value, bool): + return False + try: + parsed = float(value) + except (TypeError, ValueError): + return False + return 0 <= parsed <= 1 + + +def validate_decision_rows( + decisions: list[dict[str, Any]], + expected_case_ids: set[str], + errors: list[str], +) -> None: + decision_ids = case_ids(decisions) + add_error(errors, not any(not case_id for case_id in decision_ids), "human-adjudication decisions must include case_id for every row") + duplicates = duplicate_case_ids([case_id for case_id in decision_ids if case_id]) + add_error(errors, not duplicates, "human-adjudication decisions must not contain duplicate case_id values") + add_error( + errors, + set(decision_ids) == expected_case_ids, + "human-adjudication decisions case_id set must match adjudication pairs", + ) + for index, item in enumerate(decisions, start=1): + blocked = forbidden_decision_fields(item) + add_error( + errors, + not blocked, + f"human-adjudication decisions[{index}] must not include raw content or answer-key fields", + ) + winner = str(item.get("winner_variant", "")).strip().upper() + add_error(errors, winner in {"A", "B"}, "human-adjudication decisions must include A/B winner_variant for every case") + add_error(errors, confidence_valid(item.get("confidence")), "human-adjudication decisions confidence must be between 0 and 1") + add_error(errors, bool(str(item.get("reason", "")).strip()), "human-adjudication decisions must include reviewer reason for every case") + + +def validate_adjudicated_pairs( + pairs: list[dict[str, Any]], + expected_count: int, + errors: list[str], +) -> set[str]: + pair_ids = case_ids(pairs) + add_error(errors, len(pairs) == expected_count, "human-adjudication adjudication pairs length must equal summary.pair_count") + add_error(errors, not any(not case_id for case_id in pair_ids), "human-adjudication adjudication pairs must include case_id") + duplicates = duplicate_case_ids([case_id for case_id in pair_ids if case_id]) + add_error(errors, not duplicates, "human-adjudication adjudication pairs must not contain duplicate case_id values") + for item in pairs: + case_id = str(item.get("case_id", "")).strip() + status = str(item.get("status", "")).strip() + add_error( + errors, + status in {"match", "disagree"}, + f"human-adjudication adjudication pair {case_id or ''} must be match or disagree", + ) + add_error( + errors, + item.get("expected_revealed") is True, + f"human-adjudication adjudication pair {case_id or ''} must reveal expected winner only after valid decision", + ) + add_error( + errors, + str(item.get("reviewer_winner_variant", "")).strip().upper() in {"A", "B"}, + f"human-adjudication adjudication pair {case_id or ''} must include reviewer_winner_variant", + ) + return {case_id for case_id in pair_ids if case_id} + + +def validate_human_adjudication_report( + adjudication: dict[str, Any], + decisions: dict[str, Any], + provenance: dict[str, Any], + errors: list[str], +) -> None: + summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {} + pair_count = real_int(summary.get("pair_count")) + judgment_count = real_int(summary.get("judgment_count")) + pending_count = real_int(summary.get("pending_count")) + invalid_decision_count = real_int(summary.get("invalid_decision_count")) + answer_revealed_count = real_int(summary.get("answer_revealed_count")) + pending_answer_hidden_count = real_int(summary.get("pending_answer_hidden_count")) + checklist_ready_count = real_int(summary.get("reviewer_checklist_ready_count")) + checklist_count = real_int(summary.get("reviewer_checklist_count")) + + add_error(errors, adjudication.get("ok") is True, "human-adjudication adjudication report ok must be true") + add_error(errors, bool(pair_count and pair_count > 0), "human-adjudication adjudication summary.pair_count must be >0") + add_error( + errors, + bool(pair_count and judgment_count == pair_count), + "human-adjudication adjudication summary.judgment_count must equal summary.pair_count", + ) + add_error(errors, pending_count == 0, "human-adjudication adjudication summary.pending_count must be 0") + add_error(errors, invalid_decision_count == 0, "human-adjudication adjudication summary.invalid_decision_count must be 0") + add_error( + errors, + bool(pair_count and answer_revealed_count == pair_count), + "human-adjudication adjudication summary.answer_revealed_count must equal summary.pair_count", + ) + add_error(errors, pending_answer_hidden_count == 0, "human-adjudication adjudication summary.pending_answer_hidden_count must be 0") + add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false") + if checklist_count is not None or checklist_ready_count is not None: + add_error( + errors, + bool(pair_count and checklist_count == pair_count and checklist_ready_count == pair_count), + "human-adjudication reviewer checklist must be ready for every pair", + ) + add_error(errors, not adjudication.get("failures"), "human-adjudication adjudication failures must be empty") + + decision_rows = object_list(decisions.get("decisions", [])) + add_error(errors, decisions.get("schema_version") == "1.0", "human-adjudication decisions.schema_version must be 1.0") + reviewer = str(decisions.get("reviewer", "")).strip() + reviewed_at = str(decisions.get("reviewed_at", "")).strip() + require_real_text(errors, reviewer, "human-adjudication decisions.reviewer") + add_error( + errors, + bool(REVIEWED_AT_RE.match(reviewed_at)), + "human-adjudication decisions.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + add_error( + errors, + bool(pair_count and len(decision_rows) == pair_count), + "human-adjudication decisions count must equal adjudication summary.pair_count", + ) + + expected_case_ids = validate_adjudicated_pairs(object_list(adjudication.get("pairs", [])), int(pair_count or 0), errors) + validate_decision_rows(decision_rows, expected_case_ids, errors) + + provenance_reviewer = str(provenance.get("reviewer", "")).strip() + add_error( + errors, + bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer), + "human-adjudication provenance.reviewer must match decisions.reviewer", + ) + add_error(errors, adjudication.get("reviewer") == reviewer, "human-adjudication adjudication reviewer must match decisions.reviewer") + add_error( + errors, + adjudication.get("reviewed_at") == reviewed_at, + "human-adjudication adjudication reviewed_at must match decisions.reviewed_at", + ) diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index b438838e..4c0fa2cf 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -13,6 +13,7 @@ TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" sys.path.insert(0, str(ROOT / "scripts")) from world_class_evidence_contract import validate_payload # noqa: E402 +from world_class_human_fixtures import assert_human_contract_artifact_validation # noqa: E402 def sha256_file(path: Path) -> str: @@ -288,160 +289,6 @@ def write_json(path: Path, payload: dict) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") -def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer: str = "Yao QA") -> None: - reviewed_at = "2026-06-14" - if complete: - summary = { - "pair_count": 2, - "judgment_count": 2, - "pending_count": 0, - "agreement_count": 2, - "disagreement_count": 0, - "invalid_decision_count": 0, - "answer_revealed_count": 2, - "pending_answer_hidden_count": 0, - "agreement_rate": 100.0, - "needs_review": False, - "failure_count": 0, - } - decisions = [ - {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, - {"case_id": "case-b", "winner_variant": "B", "confidence": 0.8, "reason": "Variant B handles the boundary."}, - ] - else: - summary = { - "pair_count": 2, - "judgment_count": 1, - "pending_count": 1, - "agreement_count": 1, - "disagreement_count": 0, - "invalid_decision_count": 0, - "answer_revealed_count": 1, - "pending_answer_hidden_count": 1, - "agreement_rate": 100.0, - "needs_review": True, - "failure_count": 0, - } - decisions = [ - {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, - {"case_id": "case-b", "winner_variant": "", "confidence": None, "reason": ""}, - ] - write_json( - skill_root / "reports" / "output_review_adjudication.json", - { - "schema_version": "1.0", - "ok": True, - "summary": summary, - "reviewer": reviewer, - "reviewed_at": reviewed_at, - "pairs": [], - "failures": [], - }, - ) - write_json( - skill_root / "reports" / "output_review_decisions.json", - { - "schema_version": "1.0", - "reviewer": reviewer, - "reviewed_at": reviewed_at, - "decisions": decisions, - }, - ) - - -def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict: - return { - "schema_version": "1.0", - "evidence_key": "human-adjudication", - "template_only": False, - "category": "human", - "source_type": "blind-ab-review", - "submitted_by": reviewer, - "submitted_at": "2026-06-14", - "summary": "Completed blind A/B reviewer decisions for ledger review.", - "artifact_refs": [ - { - "path": "reports/output_review_adjudication.json", - "kind": "adjudication-report", - "contains_raw_content": False, - "sha256": sha256_file(skill_root / "reports" / "output_review_adjudication.json"), - }, - { - "path": "reports/output_review_decisions.json", - "kind": "review-decisions", - "contains_raw_content": False, - "sha256": sha256_file(skill_root / "reports" / "output_review_decisions.json"), - }, - ], - "provenance": { - "reviewer": reviewer, - "blind_pack_path": "reports/output_blind_review_pack.md", - "answer_key_opened_after_decisions": True, - }, - "privacy": { - "raw_user_content_included": False, - "raw_provider_prompt_included": False, - "credentials_included": False, - "secrets_included": False, - }, - "anti_overclaim": { - "planned_work_counts_as_evidence": False, - "metadata_fallback_counts_as_native_enforcement": False, - "pending_review_counts_as_human_decision": False, - "local_command_runner_counts_as_provider_model": False, - }, - "attestation": { - "real_external_or_human_evidence": True, - "reviewer_or_operator_identity_present": True, - "artifact_refs_reviewed": True, - "privacy_contract_satisfied": True, - "ledger_reviewer_approved": True, - "ledger_reviewer": "Yao ledger reviewer", - "ledger_reviewed_at": "2026-06-14", - }, - } - - -def assert_human_contract_artifact_validation() -> None: - entry = {"key": "human-adjudication", "category": "human"} - skill_root = TMP / "human_contract_root" - write_human_artifacts(skill_root, complete=True) - valid_result = validate_payload( - human_submission(skill_root), - entry, - path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", - root=skill_root, - template_expected=False, - ) - assert valid_result["status"] == "pass", valid_result - assert valid_result["artifact_integrity"]["required_artifact_verified_count"] == 2, valid_result - - write_human_artifacts(skill_root, complete=False) - pending_result = validate_payload( - human_submission(skill_root), - entry, - path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", - root=skill_root, - template_expected=False, - ) - assert pending_result["status"] == "fail", pending_result - assert any("summary.pending_count must be 0" in error for error in pending_result["errors"]), pending_result["errors"] - assert any("summary.judgment_count must equal" in error for error in pending_result["errors"]), pending_result["errors"] - assert any("summary.needs_review must be false" in error for error in pending_result["errors"]), pending_result["errors"] - assert any("A/B winner_variant" in error for error in pending_result["errors"]), pending_result["errors"] - - write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") - mismatch_result = validate_payload( - human_submission(skill_root, reviewer="Different Reviewer"), - entry, - path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", - root=skill_root, - template_expected=False, - ) - assert mismatch_result["status"] == "fail", mismatch_result - assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"] - - def assert_external_contract_artifact_validation() -> None: skill_root = TMP / "external_contract_root" provider_entry = {"key": "provider-holdout", "category": "external"} diff --git a/tests/world_class_human_fixtures.py b/tests/world_class_human_fixtures.py new file mode 100644 index 00000000..f039ff5c --- /dev/null +++ b/tests/world_class_human_fixtures.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + +from world_class_evidence_contract import validate_payload + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict: + return { + "schema_version": "1.0", + "evidence_key": "human-adjudication", + "template_only": False, + "category": "human", + "source_type": "blind-ab-review", + "submitted_by": reviewer, + "submitted_at": "2026-06-14", + "summary": "Completed blind A/B reviewer decisions for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_review_adjudication.json", + "kind": "adjudication-report", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_adjudication.json"), + }, + { + "path": "reports/output_review_decisions.json", + "kind": "review-decisions", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_decisions.json"), + }, + ], + "provenance": { + "reviewer": reviewer, + "blind_pack_path": "reports/output_blind_review_pack.md", + "answer_key_opened_after_decisions": True, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def human_pairs(*, complete: bool) -> list[dict]: + if complete: + return [ + { + "case_id": "case-a", + "status": "match", + "expected_revealed": True, + "reviewer_winner_variant": "A", + "confidence": 0.9, + "reason": "Variant A follows the rubric.", + }, + { + "case_id": "case-b", + "status": "disagree", + "expected_revealed": True, + "reviewer_winner_variant": "B", + "confidence": 0.8, + "reason": "Variant B handles the boundary.", + }, + ] + return [ + { + "case_id": "case-a", + "status": "match", + "expected_revealed": True, + "reviewer_winner_variant": "A", + "confidence": 0.9, + "reason": "Variant A follows the rubric.", + }, + { + "case_id": "case-b", + "status": "pending", + "expected_revealed": False, + "reviewer_winner_variant": "", + "confidence": None, + "reason": "", + }, + ] + + +def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer: str = "Yao QA") -> None: + reviewed_at = "2026-06-14" + summary = { + "pair_count": 2, + "judgment_count": 2 if complete else 1, + "pending_count": 0 if complete else 1, + "agreement_count": 2 if complete else 1, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 2 if complete else 1, + "pending_answer_hidden_count": 0 if complete else 1, + "agreement_rate": 100.0, + "needs_review": not complete, + "failure_count": 0, + } + decisions = [ + {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, + { + "case_id": "case-b", + "winner_variant": "B" if complete else "", + "confidence": 0.8 if complete else None, + "reason": "Variant B handles the boundary." if complete else "", + }, + ] + write_json( + skill_root / "reports" / "output_review_adjudication.json", + { + "schema_version": "1.0", + "ok": True, + "summary": summary, + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "pairs": human_pairs(complete=complete), + "failures": [], + }, + ) + write_json( + skill_root / "reports" / "output_review_decisions.json", + { + "schema_version": "1.0", + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "decisions": decisions, + }, + ) + + +def assert_human_submission_error(skill_root: Path, entry: dict, expected_error: str) -> None: + result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert result["status"] == "fail", result + assert any(expected_error in error for error in result["errors"]), result["errors"] + + +def assert_human_contract_artifact_validation() -> None: + entry = {"key": "human-adjudication", "category": "human"} + skill_root = TMP / "human_contract_root" + write_human_artifacts(skill_root, complete=True) + valid_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert valid_result["status"] == "pass", valid_result + assert valid_result["artifact_integrity"]["required_artifact_verified_count"] == 2, valid_result + + write_human_artifacts(skill_root, complete=False) + pending_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert pending_result["status"] == "fail", pending_result + assert any("summary.pending_count must be 0" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.judgment_count must equal" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.needs_review must be false" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("A/B winner_variant" in error for error in pending_result["errors"]), pending_result["errors"] + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + mismatch_result = validate_payload( + human_submission(skill_root, reviewer="Different Reviewer"), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert mismatch_result["status"] == "fail", mismatch_result + assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"] + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][1]["case_id"] = "unknown-case" + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "case_id set must match adjudication pairs") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["confidence"] = 1.2 + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "confidence must be between 0 and 1") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["expected_winner_variant"] = "A" + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "answer-key fields")