From 96d1fbe2094fd64d20bcf8cbaca2821065c5446e Mon Sep 17 00:00:00 2001 From: yaojingang Date: Wed, 17 Jun 2026 02:34:17 +0800 Subject: [PATCH] Tighten human evidence intake validation --- .../templates/human-adjudication.intake.json | 1 + scripts/world_class_evidence_contract.py | 11 ++++ scripts/world_class_human_evidence.py | 38 +++++++++++++ tests/verify_world_class_evidence_intake.py | 1 + tests/world_class_human_fixtures.py | 57 +++++++++++++++++++ 5 files changed, 108 insertions(+) diff --git a/evidence/world_class/templates/human-adjudication.intake.json b/evidence/world_class/templates/human-adjudication.intake.json index 14bf4bf..1ac1cb0 100644 --- a/evidence/world_class/templates/human-adjudication.intake.json +++ b/evidence/world_class/templates/human-adjudication.intake.json @@ -23,6 +23,7 @@ ], "provenance": { "reviewer": "name or team handle", + "reviewed_at": "YYYY-MM-DD", "blind_pack_path": "reports/output_blind_review_pack.md", "answer_key_opened_after_decisions": true, "decision_fields": [ diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index 1010e5b..0f2f37a 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -404,13 +404,24 @@ def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> No ) elif key == "human-adjudication": add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required") + add_error(errors, bool(str(provenance.get("reviewed_at", "")).strip()), "human-adjudication provenance.reviewed_at is required") if not template_expected: require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer") + add_error( + errors, + bool(SUBMITTED_AT_RE.match(str(provenance.get("reviewed_at", "")).strip())), + "human-adjudication provenance.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) add_error( errors, provenance.get("answer_key_opened_after_decisions") is True, "human-adjudication must attest answer_key_opened_after_decisions is true", ) + add_error( + errors, + provenance.get("reviewer_reason_required") is True, + "human-adjudication must attest reviewer_reason_required is true", + ) elif key == "native-permission-enforcement": add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required") if not template_expected: diff --git a/scripts/world_class_human_evidence.py b/scripts/world_class_human_evidence.py index a6fc0df..6123388 100644 --- a/scripts/world_class_human_evidence.py +++ b/scripts/world_class_human_evidence.py @@ -145,6 +145,21 @@ def validate_human_adjudication_report( ) add_error(errors, pending_answer_hidden_count == 0, "human-adjudication adjudication summary.pending_answer_hidden_count must be 0") add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false") + add_error( + errors, + summary.get("reviewer_metadata_present") is True, + "human-adjudication adjudication summary.reviewer_metadata_present must be true", + ) + add_error( + errors, + summary.get("reason_required") is True, + "human-adjudication adjudication summary.reason_required must be true", + ) + add_error( + errors, + summary.get("ready_for_human_evidence") is True, + "human-adjudication adjudication summary.ready_for_human_evidence must be true", + ) if checklist_count is not None or checklist_ready_count is not None: add_error( errors, @@ -173,11 +188,34 @@ def validate_human_adjudication_report( validate_decision_rows(decision_rows, expected_case_ids, errors) provenance_reviewer = str(provenance.get("reviewer", "")).strip() + provenance_reviewed_at = str(provenance.get("reviewed_at", "")).strip() add_error( errors, bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer), "human-adjudication provenance.reviewer must match decisions.reviewer", ) + add_error( + errors, + bool(reviewed_at and provenance_reviewed_at and provenance_reviewed_at == reviewed_at), + "human-adjudication provenance.reviewed_at must match decisions.reviewed_at", + ) + add_error( + errors, + provenance.get("answer_key_opened_after_decisions") is True, + "human-adjudication provenance.answer_key_opened_after_decisions must be true", + ) + add_error( + errors, + provenance.get("reviewer_reason_required") is True, + "human-adjudication provenance.reviewer_reason_required must be true", + ) + decision_fields = provenance.get("decision_fields", []) + required_decision_fields = {"case_id", "winner_variant", "reason"} + add_error( + errors, + isinstance(decision_fields, list) and required_decision_fields <= {str(item) for item in decision_fields}, + "human-adjudication provenance.decision_fields must include case_id, winner_variant, and reason", + ) add_error(errors, adjudication.get("reviewer") == reviewer, "human-adjudication adjudication reviewer must match decisions.reviewer") add_error( errors, diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 276d7cf..34b33c0 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -502,6 +502,7 @@ def main() -> None: human_template = json.loads((ROOT / "evidence" / "world_class" / "templates" / "human-adjudication.intake.json").read_text(encoding="utf-8")) decision_ref = next(item for item in human_template["artifact_refs"] if item["path"] == "reports/output_review_decisions.json") assert "required reason" in decision_ref["note"], human_template + assert human_template["provenance"]["reviewed_at"] == "YYYY-MM-DD", human_template assert human_template["provenance"]["reviewer_reason_required"] is True, human_template assert "reason" in human_template["provenance"]["decision_fields"], human_template assert "visible rubric" in human_template["privacy"]["notes"], human_template diff --git a/tests/world_class_human_fixtures.py b/tests/world_class_human_fixtures.py index 2c475d3..7c71944 100644 --- a/tests/world_class_human_fixtures.py +++ b/tests/world_class_human_fixtures.py @@ -49,8 +49,11 @@ def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict: ], "provenance": { "reviewer": reviewer, + "reviewed_at": "2026-06-14", "blind_pack_path": "reports/output_blind_review_pack.md", "answer_key_opened_after_decisions": True, + "decision_fields": ["case_id", "winner_variant", "confidence", "reason"], + "reviewer_reason_required": True, }, "privacy": { "raw_user_content_included": False, @@ -130,6 +133,11 @@ def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer: "agreement_rate": 100.0, "needs_review": not complete, "failure_count": 0, + "reviewer_metadata_present": complete, + "reason_required": True, + "ready_for_human_evidence": complete, + "reviewer_checklist_count": 2, + "reviewer_checklist_ready_count": 2 if complete else 1, } decisions = [ {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, @@ -226,6 +234,55 @@ def assert_human_contract_artifact_validation() -> None: assert mismatch_result["status"] == "fail", mismatch_result assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"] + reviewed_at_mismatch = human_submission(skill_root, reviewer="Yao QA") + reviewed_at_mismatch["provenance"]["reviewed_at"] = "2026-06-15" + reviewed_at_result = validate_payload( + reviewed_at_mismatch, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert reviewed_at_result["status"] == "fail", reviewed_at_result + assert any("provenance.reviewed_at must match decisions.reviewed_at" in error for error in reviewed_at_result["errors"]), ( + reviewed_at_result["errors"] + ) + + answer_key_order = human_submission(skill_root, reviewer="Yao QA") + answer_key_order["provenance"]["answer_key_opened_after_decisions"] = False + answer_key_result = validate_payload( + answer_key_order, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert answer_key_result["status"] == "fail", answer_key_result + assert any("answer_key_opened_after_decisions" in error for error in answer_key_result["errors"]), answer_key_result["errors"] + + missing_reason_contract = human_submission(skill_root, reviewer="Yao QA") + missing_reason_contract["provenance"]["reviewer_reason_required"] = False + missing_reason_contract["provenance"]["decision_fields"] = ["case_id", "winner_variant", "confidence"] + reason_contract_result = validate_payload( + missing_reason_contract, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert reason_contract_result["status"] == "fail", reason_contract_result + assert any("reviewer_reason_required" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"] + assert any("decision_fields must include" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"] + + forged_adjudication = json.loads((skill_root / "reports" / "output_review_adjudication.json").read_text(encoding="utf-8")) + forged_adjudication["summary"]["ready_for_human_evidence"] = False + forged_adjudication["summary"]["reviewer_metadata_present"] = False + forged_adjudication["summary"]["reason_required"] = False + write_json(skill_root / "reports" / "output_review_adjudication.json", forged_adjudication) + assert_human_submission_error(skill_root, entry, "summary.ready_for_human_evidence must be true") + assert_human_submission_error(skill_root, entry, "summary.reviewer_metadata_present must be true") + assert_human_submission_error(skill_root, entry, "summary.reason_required must be true") + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) forged_decisions["decisions"][1]["case_id"] = "unknown-case"