From 9b9ea7d4fb43e2bc24f69a09df26b79e084fc7bb Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 03:25:39 +0800 Subject: [PATCH] Harden human adjudication evidence intake --- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/output_execution_runs.json | 20 +-- reports/output_execution_runs.md | 20 +-- reports/package_verification.json | 2 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/upgrade_check.json | 4 +- scripts/world_class_evidence_contract.py | 120 +++++++++++++++ scripts/yao_cli_report_commands.py | 3 + tests/verify_world_class_evidence_intake.py | 160 ++++++++++++++++++++ tests/verify_yao_cli.py | 2 +- 15 files changed, 320 insertions(+), 37 deletions(-) diff --git a/registry/index.json b/registry/index.json index 8a6387a0..bcf1ffd9 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c" + "package_sha256": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 709644c9..bb582729 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c", - "archive_sha256": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f" + "package_sha256": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf", + "archive_sha256": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f", + "archive_sha256": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index fde99a60..b94f969c 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.19, + "duration_ms": 27.59, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.21, + "duration_ms": 26.56, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.21, + "duration_ms": 27.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.35, + "duration_ms": 29.17, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.03, + "duration_ms": 29.81, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.51, + "duration_ms": 27.82, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.42, + "duration_ms": 26.57, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.03, + "duration_ms": 26.29, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.05, + "duration_ms": 26.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.28, + "duration_ms": 26.13, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index ca2aa8fc..88298a9c 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.19 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.21 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 29.21 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.35 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 30.03 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.51 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.42 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.03 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.05 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.28 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.59 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.56 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.04 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.17 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.81 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.82 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.57 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.29 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.04 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.13 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index df87e4a7..b9474bc1 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f", + "archive_sha256": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69", "archive_entry_count": 626, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index 36c40d2f..41b939b8 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f` +- Archive SHA256: `56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index becb4e6c..98aa24a5 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c", - "archive_sha256": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f" + "package_sha256": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf", + "archive_sha256": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f", + "archive_sha256": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c" + "package_sha256": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 2c7da6ab..b6895874 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c` -- Archive SHA256: `ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f` +- Package SHA256: `f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf` +- Archive SHA256: `56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index bcf1f39a..e0a21669 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 201, - "package_sha256": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c" + "package_sha256": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 8943315c..60f4c533 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `201` -- Package SHA256: `836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c` +- Package SHA256: `f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf` ## Failures diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index acb5d263..ee158e59 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "ed1bc47c56c8d247bf5dd351bf29201e60471b4785a86e16ae1ede5e71d1582f" + "to": "56ae3ef2ba798437b7253add936fc87ca34da09890359f6f53b7d6e775093d69" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "836b5ebaf4a608a1a89fa84306ab9b4bb7e0555c74b6ca8e572ffb1833cc0f8c" + "to": "f088abea9460cfb91e969de1397dd16e9e1a5ba18ad96db4c2a7acad0d34dcbf" } ] }, diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index ae057061..1e53e6f0 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -222,6 +222,125 @@ def validate_artifact_refs( return stats +def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path]: + refs = payload.get("artifact_refs", []) + if not isinstance(refs, list): + return {} + paths: dict[str, Path] = {} + for ref in refs: + if not isinstance(ref, dict): + continue + path_text = str(ref.get("path", "")).strip() + if not path_text: + continue + resolved, path_error = resolve_artifact_path(path_text, root) + if path_error or resolved is None: + continue + paths[rel_path(resolved, root)] = resolved + return paths + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) else None + + +def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: + paths = artifact_ref_path_map(payload, root) + adjudication = load_json(paths.get("reports/output_review_adjudication.json", root / "__missing__")) + decisions = load_json(paths.get("reports/output_review_decisions.json", root / "__missing__")) + if not adjudication or not decisions: + return + + summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {} + pair_count = real_int(summary.get("pair_count")) + judgment_count = real_int(summary.get("judgment_count")) + pending_count = real_int(summary.get("pending_count")) + invalid_decision_count = real_int(summary.get("invalid_decision_count")) + answer_revealed_count = real_int(summary.get("answer_revealed_count")) + pending_answer_hidden_count = real_int(summary.get("pending_answer_hidden_count")) + add_error(errors, bool(pair_count and pair_count > 0), "human-adjudication adjudication summary.pair_count must be >0") + add_error( + errors, + bool(pair_count and judgment_count == pair_count), + "human-adjudication adjudication summary.judgment_count must equal summary.pair_count", + ) + add_error(errors, pending_count == 0, "human-adjudication adjudication summary.pending_count must be 0") + add_error( + errors, + invalid_decision_count == 0, + "human-adjudication adjudication summary.invalid_decision_count must be 0", + ) + add_error( + errors, + bool(pair_count and answer_revealed_count == pair_count), + "human-adjudication adjudication summary.answer_revealed_count must equal summary.pair_count", + ) + add_error( + errors, + pending_answer_hidden_count == 0, + "human-adjudication adjudication summary.pending_answer_hidden_count must be 0", + ) + add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false") + + decision_rows = decisions.get("decisions", []) + add_error(errors, decisions.get("schema_version") == "1.0", "human-adjudication decisions.schema_version must be 1.0") + reviewer = str(decisions.get("reviewer", "")).strip() + reviewed_at = str(decisions.get("reviewed_at", "")).strip() + require_real_text(errors, reviewer, "human-adjudication decisions.reviewer") + add_error( + errors, + bool(SUBMITTED_AT_RE.match(reviewed_at)), + "human-adjudication decisions.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + add_error( + errors, + isinstance(decision_rows, list) and bool(pair_count and len(decision_rows) == pair_count), + "human-adjudication decisions count must equal adjudication summary.pair_count", + ) + if isinstance(decision_rows, list): + invalid_winners = [ + str(item.get("case_id", index + 1)) + for index, item in enumerate(decision_rows) + if not isinstance(item, dict) or str(item.get("winner_variant", "")).strip().upper() not in {"A", "B"} + ] + missing_reasons = [ + str(item.get("case_id", index + 1)) + for index, item in enumerate(decision_rows) + if not isinstance(item, dict) or not str(item.get("reason", "")).strip() + ] + add_error( + errors, + not invalid_winners, + "human-adjudication decisions must include A/B winner_variant for every case", + ) + add_error(errors, not missing_reasons, "human-adjudication decisions must include reviewer reason for every case") + + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + provenance_reviewer = str(provenance.get("reviewer", "")).strip() + add_error( + errors, + bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer), + "human-adjudication provenance.reviewer must match decisions.reviewer", + ) + add_error( + errors, + adjudication.get("reviewer") == reviewer, + "human-adjudication adjudication reviewer must match decisions.reviewer", + ) + add_error( + errors, + adjudication.get("reviewed_at") == reviewed_at, + "human-adjudication adjudication reviewed_at must match decisions.reviewed_at", + ) + + +def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None: + if template_expected: + return + if str(payload.get("evidence_key", "")) == "human-adjudication": + validate_human_adjudication_artifacts(payload, errors, root) + + def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None: key = str(payload.get("evidence_key", "")) template_expected = payload.get("template_only") is True @@ -331,6 +450,7 @@ def validate_payload( if not template_expected: for key in REQUIRED_ATTESTATION_TRUE: add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission") + validate_real_artifact_payloads(payload, errors, root, template_expected) return { "path": rel_path(path, root), "evidence_key": evidence_key, diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index 79710f43..4ce2bf1d 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -80,6 +80,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_benchmark_reproducibility.py", [str(ROOT)]), run_script("render_skill_overview.py", [str(ROOT)]), run_script("render_skill_interpretation.py", [str(ROOT)]), + run_script("render_review_studio.py", [str(ROOT)]), run_script("render_evidence_consistency.py", [str(ROOT)]), run_script("render_review_viewer.py", [str(ROOT)]), ] @@ -124,6 +125,8 @@ def command_report(args: argparse.Namespace) -> int: "skill_overview": "reports/skill-overview.json", "skill_interpretation": "reports/skill-interpretation.json", "skill_interpretation_html": "reports/skill-interpretation.html", + "review_studio": "reports/review-studio.json", + "review_studio_html": "reports/review-studio.html", "evidence_consistency": "reports/evidence_consistency.json", "review_viewer": "reports/review-viewer.json", "review_viewer_html": "reports/review-viewer.html", diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 871265de..b03bd6d0 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -11,6 +11,9 @@ ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py" KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py" TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" +sys.path.insert(0, str(ROOT / "scripts")) + +from world_class_evidence_contract import validate_payload # noqa: E402 def sha256_file(path: Path) -> str: @@ -100,6 +103,162 @@ def provider_submission(*, valid: bool, artifact_path: str = "reports/output_exe } +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer: str = "Yao QA") -> None: + reviewed_at = "2026-06-14" + if complete: + summary = { + "pair_count": 2, + "judgment_count": 2, + "pending_count": 0, + "agreement_count": 2, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 2, + "pending_answer_hidden_count": 0, + "agreement_rate": 100.0, + "needs_review": False, + "failure_count": 0, + } + decisions = [ + {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, + {"case_id": "case-b", "winner_variant": "B", "confidence": 0.8, "reason": "Variant B handles the boundary."}, + ] + else: + summary = { + "pair_count": 2, + "judgment_count": 1, + "pending_count": 1, + "agreement_count": 1, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 1, + "pending_answer_hidden_count": 1, + "agreement_rate": 100.0, + "needs_review": True, + "failure_count": 0, + } + decisions = [ + {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, + {"case_id": "case-b", "winner_variant": "", "confidence": None, "reason": ""}, + ] + write_json( + skill_root / "reports" / "output_review_adjudication.json", + { + "schema_version": "1.0", + "ok": True, + "summary": summary, + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "pairs": [], + "failures": [], + }, + ) + write_json( + skill_root / "reports" / "output_review_decisions.json", + { + "schema_version": "1.0", + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "decisions": decisions, + }, + ) + + +def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict: + return { + "schema_version": "1.0", + "evidence_key": "human-adjudication", + "template_only": False, + "category": "human", + "source_type": "blind-ab-review", + "submitted_by": reviewer, + "submitted_at": "2026-06-14", + "summary": "Completed blind A/B reviewer decisions for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_review_adjudication.json", + "kind": "adjudication-report", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_adjudication.json"), + }, + { + "path": "reports/output_review_decisions.json", + "kind": "review-decisions", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_decisions.json"), + }, + ], + "provenance": { + "reviewer": reviewer, + "blind_pack_path": "reports/output_blind_review_pack.md", + "answer_key_opened_after_decisions": True, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + }, + } + + +def assert_human_contract_artifact_validation() -> None: + entry = {"key": "human-adjudication", "category": "human"} + skill_root = TMP / "human_contract_root" + write_human_artifacts(skill_root, complete=True) + valid_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert valid_result["status"] == "pass", valid_result + assert valid_result["artifact_integrity"]["required_artifact_verified_count"] == 2, valid_result + + write_human_artifacts(skill_root, complete=False) + pending_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert pending_result["status"] == "fail", pending_result + assert any("summary.pending_count must be 0" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.judgment_count must equal" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.needs_review must be false" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("A/B winner_variant" in error for error in pending_result["errors"]), pending_result["errors"] + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + mismatch_result = validate_payload( + human_submission(skill_root, reviewer="Different Reviewer"), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert mismatch_result["status"] == "fail", mismatch_result + assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"] + + def assert_documented_submission_commands() -> None: expected_fragments = [ 'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"', @@ -122,6 +281,7 @@ def main() -> None: assert_documented_submission_commands() default_json = TMP / "world_class_evidence_intake.json" default_md = TMP / "world_class_evidence_intake.md" + assert_human_contract_artifact_validation() payload = run_intake("--output-json", str(default_json), "--output-md", str(default_md)) assert payload["schema_version"] == "1.0", payload assert payload["ok"] is True, payload diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 0253fb0f..a983e226 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -5,7 +5,6 @@ import subprocess import sys from pathlib import Path - ROOT = Path(__file__).resolve().parent.parent CLI = ROOT / "scripts" / "yao.py" BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" @@ -767,6 +766,7 @@ def main() -> None: assert report_result["payload"]["artifacts"]["skill_overview"] == "reports/skill-overview.json", report_result assert report_result["payload"]["artifacts"]["skill_interpretation"] == "reports/skill-interpretation.json", report_result assert report_result["payload"]["artifacts"]["skill_interpretation_html"] == "reports/skill-interpretation.html", report_result + assert (report_result["payload"]["artifacts"]["review_studio"], report_result["payload"]["artifacts"]["review_studio_html"]) == ("reports/review-studio.json", "reports/review-studio.html"), report_result assert report_result["payload"]["artifacts"]["review_viewer"] == "reports/review-viewer.json", report_result assert report_result["payload"]["artifacts"]["review_viewer_html"] == "reports/review-viewer.html", report_result report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8"))