frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Tighten human evidence intake validation

Browse Source
This commit is contained in:
yaojingang
2026-06-17 02:34:17 +08:00
parent 8c18faf49b
commit 96d1fbe209
5 changed files with 108 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evidence/world_class/templates/human-adjudication.intake.json
+1
View File
@@ -23,6 +23,7 @@
],
"provenance": {
"reviewer": "name or team handle",
"reviewed_at": "YYYY-MM-DD",
"blind_pack_path": "reports/output_blind_review_pack.md",
"answer_key_opened_after_decisions": true,
"decision_fields": [
scripts/world_class_evidence_contract.py
+11
View File
@@ -404,13 +404,24 @@ def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> No
)
elif key == "human-adjudication":
add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
add_error(errors, bool(str(provenance.get("reviewed_at", "")).strip()), "human-adjudication provenance.reviewed_at is required")
if not template_expected:
require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer")
add_error(
errors,
bool(SUBMITTED_AT_RE.match(str(provenance.get("reviewed_at", "")).strip())),
"human-adjudication provenance.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
)
add_error(
errors,
provenance.get("answer_key_opened_after_decisions") is True,
"human-adjudication must attest answer_key_opened_after_decisions is true",
)
add_error(
errors,
provenance.get("reviewer_reason_required") is True,
"human-adjudication must attest reviewer_reason_required is true",
)
elif key == "native-permission-enforcement":
add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
if not template_expected:
scripts/world_class_human_evidence.py
+38
View File
@@ -145,6 +145,21 @@ def validate_human_adjudication_report(
)
add_error(errors, pending_answer_hidden_count == 0, "human-adjudication adjudication summary.pending_answer_hidden_count must be 0")
add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false")
add_error(
errors,
summary.get("reviewer_metadata_present") is True,
"human-adjudication adjudication summary.reviewer_metadata_present must be true",
)
add_error(
errors,
summary.get("reason_required") is True,
"human-adjudication adjudication summary.reason_required must be true",
)
add_error(
errors,
summary.get("ready_for_human_evidence") is True,
"human-adjudication adjudication summary.ready_for_human_evidence must be true",
)
if checklist_count is not None or checklist_ready_count is not None:
add_error(
errors,
@@ -173,11 +188,34 @@ def validate_human_adjudication_report(
validate_decision_rows(decision_rows, expected_case_ids, errors)
provenance_reviewer = str(provenance.get("reviewer", "")).strip()
provenance_reviewed_at = str(provenance.get("reviewed_at", "")).strip()
add_error(
errors,
bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer),
"human-adjudication provenance.reviewer must match decisions.reviewer",
)
add_error(
errors,
bool(reviewed_at and provenance_reviewed_at and provenance_reviewed_at == reviewed_at),
"human-adjudication provenance.reviewed_at must match decisions.reviewed_at",
)
add_error(
errors,
provenance.get("answer_key_opened_after_decisions") is True,
"human-adjudication provenance.answer_key_opened_after_decisions must be true",
)
add_error(
errors,
provenance.get("reviewer_reason_required") is True,
"human-adjudication provenance.reviewer_reason_required must be true",
)
decision_fields = provenance.get("decision_fields", [])
required_decision_fields = {"case_id", "winner_variant", "reason"}
add_error(
errors,
isinstance(decision_fields, list) and required_decision_fields <= {str(item) for item in decision_fields},
"human-adjudication provenance.decision_fields must include case_id, winner_variant, and reason",
)
add_error(errors, adjudication.get("reviewer") == reviewer, "human-adjudication adjudication reviewer must match decisions.reviewer")
add_error(
errors,
tests/verify_world_class_evidence_intake.py
+1
View File
@@ -502,6 +502,7 @@ def main() -> None:
human_template = json.loads((ROOT / "evidence" / "world_class" / "templates" / "human-adjudication.intake.json").read_text(encoding="utf-8"))
decision_ref = next(item for item in human_template["artifact_refs"] if item["path"] == "reports/output_review_decisions.json")
assert "required reason" in decision_ref["note"], human_template
assert human_template["provenance"]["reviewed_at"] == "YYYY-MM-DD", human_template
assert human_template["provenance"]["reviewer_reason_required"] is True, human_template
assert "reason" in human_template["provenance"]["decision_fields"], human_template
assert "visible rubric" in human_template["privacy"]["notes"], human_template
tests/world_class_human_fixtures.py
+57
View File
@@ -49,8 +49,11 @@ def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict:
],
"provenance": {
"reviewer": reviewer,
"reviewed_at": "2026-06-14",
"blind_pack_path": "reports/output_blind_review_pack.md",
"answer_key_opened_after_decisions": True,
"decision_fields": ["case_id", "winner_variant", "confidence", "reason"],
"reviewer_reason_required": True,
},
"privacy": {
"raw_user_content_included": False,
@@ -130,6 +133,11 @@ def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer:
"agreement_rate": 100.0,
"needs_review": not complete,
"failure_count": 0,
"reviewer_metadata_present": complete,
"reason_required": True,
"ready_for_human_evidence": complete,
"reviewer_checklist_count": 2,
"reviewer_checklist_ready_count": 2 if complete else 1,
}
decisions = [
{"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."},
@@ -226,6 +234,55 @@ def assert_human_contract_artifact_validation() -> None:
assert mismatch_result["status"] == "fail", mismatch_result
assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"]
reviewed_at_mismatch = human_submission(skill_root, reviewer="Yao QA")
reviewed_at_mismatch["provenance"]["reviewed_at"] = "2026-06-15"
reviewed_at_result = validate_payload(
reviewed_at_mismatch,
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json",
root=skill_root,
template_expected=False,
)
assert reviewed_at_result["status"] == "fail", reviewed_at_result
assert any("provenance.reviewed_at must match decisions.reviewed_at" in error for error in reviewed_at_result["errors"]), (
reviewed_at_result["errors"]
)
answer_key_order = human_submission(skill_root, reviewer="Yao QA")
answer_key_order["provenance"]["answer_key_opened_after_decisions"] = False
answer_key_result = validate_payload(
answer_key_order,
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json",
root=skill_root,
template_expected=False,
)
assert answer_key_result["status"] == "fail", answer_key_result
assert any("answer_key_opened_after_decisions" in error for error in answer_key_result["errors"]), answer_key_result["errors"]
missing_reason_contract = human_submission(skill_root, reviewer="Yao QA")
missing_reason_contract["provenance"]["reviewer_reason_required"] = False
missing_reason_contract["provenance"]["decision_fields"] = ["case_id", "winner_variant", "confidence"]
reason_contract_result = validate_payload(
missing_reason_contract,
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json",
root=skill_root,
template_expected=False,
)
assert reason_contract_result["status"] == "fail", reason_contract_result
assert any("reviewer_reason_required" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"]
assert any("decision_fields must include" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"]
forged_adjudication = json.loads((skill_root / "reports" / "output_review_adjudication.json").read_text(encoding="utf-8"))
forged_adjudication["summary"]["ready_for_human_evidence"] = False
forged_adjudication["summary"]["reviewer_metadata_present"] = False
forged_adjudication["summary"]["reason_required"] = False
write_json(skill_root / "reports" / "output_review_adjudication.json", forged_adjudication)
assert_human_submission_error(skill_root, entry, "summary.ready_for_human_evidence must be true")
assert_human_submission_error(skill_root, entry, "summary.reviewer_metadata_present must be true")
assert_human_submission_error(skill_root, entry, "summary.reason_required must be true")
write_human_artifacts(skill_root, complete=True, reviewer="Yao QA")
forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8"))
forged_decisions["decisions"][1]["case_id"] = "unknown-case"