yao-meta-skill/scripts/adjudicate_output_review.py

#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Any


ROOT = Path(__file__).resolve().parent.parent
DEFAULT_BLIND_PACK = ROOT / "reports" / "output_blind_review_pack.json"
DEFAULT_ANSWER_KEY = ROOT / "reports" / "output_blind_answer_key.json"
DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json"
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_adjudication.json"
DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_adjudication.md"


def display_path(path: Path) -> str:
    try:
        return str(path.resolve().relative_to(ROOT.resolve()))
    except ValueError:
        return str(path.resolve())


def load_json(path: Path) -> tuple[dict[str, Any], list[str]]:
    if not path.exists():
        return {}, [f"Missing JSON file: {display_path(path)}"]
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        return {}, [f"Invalid JSON file {display_path(path)}: {exc}"]
    if not isinstance(payload, dict):
        return {}, [f"JSON root must be an object: {display_path(path)}"]
    return payload, []


def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]:
    if not path.exists():
        return {"schema_version": "1.0", "decisions": []}, []
    return load_json(path)


def normalize_variant(value: Any) -> str:
    normalized = str(value or "").strip().upper()
    if normalized in {"VARIANT A", "A"}:
        return "A"
    if normalized in {"VARIANT B", "B"}:
        return "B"
    return normalized


def confidence_value(value: Any) -> tuple[float | None, str | None]:
    if value is None or value == "":
        return None, None
    try:
        parsed = float(value)
    except (TypeError, ValueError):
        return None, f"confidence must be numeric, got {value!r}"
    if parsed < 0 or parsed > 1:
        return None, f"confidence must be between 0 and 1, got {parsed}"
    return round(parsed, 3), None


def answer_index(answer_key: dict[str, Any]) -> dict[str, dict[str, Any]]:
    answers = answer_key.get("answers", [])
    if not isinstance(answers, list):
        return {}
    return {str(item.get("case_id", "")): item for item in answers if isinstance(item, dict) and item.get("case_id")}


def pair_index(blind_pack: dict[str, Any]) -> dict[str, dict[str, Any]]:
    pairs = blind_pack.get("pairs", [])
    if not isinstance(pairs, list):
        return {}
    return {str(item.get("case_id", "")): item for item in pairs if isinstance(item, dict) and item.get("case_id")}


def decision_index(decisions_payload: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], list[str]]:
    decisions = decisions_payload.get("decisions", [])
    failures: list[str] = []
    indexed: dict[str, dict[str, Any]] = {}
    if not isinstance(decisions, list):
        return {}, ["decisions must be a list"]
    for index, item in enumerate(decisions, start=1):
        if not isinstance(item, dict):
            failures.append(f"decision #{index} must be an object")
            continue
        case_id = str(item.get("case_id", "")).strip()
        if not case_id:
            failures.append(f"decision #{index} is missing case_id")
            continue
        if case_id in indexed:
            failures.append(f"duplicate decision for case_id: {case_id}")
            continue
        indexed[case_id] = item
    return indexed, failures


def build_decision_template(blind_pack: dict[str, Any]) -> dict[str, Any]:
    pairs = blind_pack.get("pairs", [])
    template_decisions = []
    if isinstance(pairs, list):
        for pair in pairs:
            if not isinstance(pair, dict):
                continue
            template_decisions.append(
                {
                    "case_id": str(pair.get("case_id", "")),
                    "winner_variant": "",
                    "confidence": None,
                    "reason": "",
                }
            )
    return {
        "schema_version": "1.0",
        "reviewer": "",
        "reviewed_at": "",
        "decision_contract": {
            "winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.",
            "confidence": "Optional number from 0 to 1.",
            "reason": "Short reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.",
        },
        "decisions": template_decisions,
    }


def adjudicate_pair(
    case_id: str,
    pair: dict[str, Any],
    answer: dict[str, Any] | None,
    decision: dict[str, Any] | None,
) -> tuple[dict[str, Any], list[str]]:
    failures: list[str] = []
    expected = normalize_variant(answer.get("expected_winner_variant", "") if answer else "")
    if expected not in {"A", "B"}:
        failures.append(f"{case_id}: answer key is missing expected_winner_variant")
    if decision is None:
        return (
            {
                "case_id": case_id,
                "status": "pending",
                "expected_winner_variant": expected,
                "reviewer_winner_variant": "",
                "confidence": None,
                "reason": "",
                "prompt": str(pair.get("prompt", "")),
            },
            failures,
        )

    reviewer = normalize_variant(
        decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", "")))
    )
    confidence, confidence_failure = confidence_value(decision.get("confidence"))
    reason = str(decision.get("reason", "")).strip()
    if not reviewer:
        return (
            {
                "case_id": case_id,
                "status": "pending",
                "expected_winner_variant": expected,
                "reviewer_winner_variant": "",
                "confidence": confidence,
                "reason": reason,
                "prompt": str(pair.get("prompt", "")),
            },
            failures,
        )
    if reviewer not in {"A", "B"}:
        failures.append(f"{case_id}: winner_variant must be A or B")
        status = "invalid"
    elif confidence_failure:
        failures.append(f"{case_id}: {confidence_failure}")
        status = "invalid"
    else:
        status = "match" if reviewer == expected else "disagree"
    return (
        {
            "case_id": case_id,
            "status": status,
            "expected_winner_variant": expected,
            "reviewer_winner_variant": reviewer,
            "confidence": confidence,
            "reason": reason,
            "prompt": str(pair.get("prompt", "")),
        },
        failures,
    )


def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]:
    pair_count = len(pairs)
    judgment_count = sum(1 for item in pairs if item["status"] in {"match", "disagree"})
    pending_count = sum(1 for item in pairs if item["status"] == "pending")
    agreement_count = sum(1 for item in pairs if item["status"] == "match")
    disagreement_count = sum(1 for item in pairs if item["status"] == "disagree")
    invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid")
    agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None
    return {
        "pair_count": pair_count,
        "judgment_count": judgment_count,
        "pending_count": pending_count,
        "agreement_count": agreement_count,
        "disagreement_count": disagreement_count,
        "invalid_decision_count": invalid_decision_count,
        "agreement_rate": agreement_rate,
        "needs_review": pending_count > 0,
        "failure_count": len(failures),
    }


def render_markdown(payload: dict[str, Any]) -> str:
    summary = payload["summary"]
    lines = [
        "# Output Review Adjudication",
        "",
        "This report adjudicates reviewer choices from the blind A/B output review pack against the separate answer key.",
        "",
        f"- Pairs: `{summary['pair_count']}`",
        f"- Judgments: `{summary['judgment_count']}`",
        f"- Pending: `{summary['pending_count']}`",
        f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`",
        f"- Invalid decisions: `{summary['invalid_decision_count']}`",
        "",
    ]
    if summary["judgment_count"] == 0:
        lines.extend(
            [
                "No reviewer decisions recorded yet.",
                "",
                "Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.",
                "",
            ]
        )
    lines.extend(
        [
            "## Case Adjudication",
            "",
            "| Case | Reviewer | Expected | Status | Confidence | Reason |",
            "| --- | --- | --- | --- | ---: | --- |",
        ]
    )
    for item in payload["pairs"]:
        confidence = "" if item.get("confidence") is None else str(item["confidence"])
        reason = str(item.get("reason", "")).replace("|", "\\|") or ""
        lines.append(
            f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | "
            f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |"
        )
    if payload.get("failures"):
        lines.extend(["", "## Failures", ""])
        for failure in payload["failures"]:
            lines.append(f"- {failure}")
    lines.extend(
        [
            "",
            "## Next Fixes",
            "",
            "- Keep the blind review pack separate from the answer key until decisions are recorded.",
            "- Treat disagreement cases as prompts for rubric tuning or output improvement.",
            "- Add model-executed holdout runs after this human adjudication harness is stable.",
        ]
    )
    return "\n".join(lines).strip() + "\n"


def adjudicate_output_review(
    blind_pack_path: Path,
    answer_key_path: Path,
    decisions_path: Path,
    output_json: Path,
    output_md: Path,
    write_template: bool = False,
) -> dict[str, Any]:
    blind_pack, failures = load_json(blind_pack_path)
    answer_key, answer_failures = load_json(answer_key_path)
    failures.extend(answer_failures)
    decisions_payload, decision_failures = load_optional_decisions(decisions_path)
    failures.extend(decision_failures)

    template_written = False
    if write_template and blind_pack and not decisions_path.exists():
        decisions_path.parent.mkdir(parents=True, exist_ok=True)
        decisions_path.write_text(
            json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        template_written = True

    pairs_by_id = pair_index(blind_pack)
    answers_by_id = answer_index(answer_key)
    decisions_by_id, index_failures = decision_index(decisions_payload)
    failures.extend(index_failures)

    for case_id in decisions_by_id:
        if case_id not in pairs_by_id:
            failures.append(f"decision references unknown case_id: {case_id}")

    adjudicated_pairs: list[dict[str, Any]] = []
    for case_id, pair in pairs_by_id.items():
        adjudicated, pair_failures = adjudicate_pair(case_id, pair, answers_by_id.get(case_id), decisions_by_id.get(case_id))
        adjudicated_pairs.append(adjudicated)
        failures.extend(pair_failures)

    summary = build_summary(adjudicated_pairs, failures)
    payload = {
        "schema_version": "1.0",
        "ok": not failures,
        "summary": summary,
        "reviewer": decisions_payload.get("reviewer", ""),
        "reviewed_at": decisions_payload.get("reviewed_at", ""),
        "artifacts": {
            "blind_pack": display_path(blind_pack_path),
            "answer_key": display_path(answer_key_path),
            "decisions": display_path(decisions_path),
            "json": display_path(output_json),
            "markdown": display_path(output_md),
        },
        "template_written": template_written,
        "pairs": adjudicated_pairs,
        "failures": failures,
    }
    output_json.parent.mkdir(parents=True, exist_ok=True)
    output_md.parent.mkdir(parents=True, exist_ok=True)
    output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    output_md.write_text(render_markdown(payload), encoding="utf-8")
    return payload


def main() -> None:
    parser = argparse.ArgumentParser(description="Adjudicate blind A/B output review decisions against the answer key.")
    parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK))
    parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY))
    parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS))
    parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON))
    parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD))
    parser.add_argument("--write-template", action="store_true")
    args = parser.parse_args()

    payload = adjudicate_output_review(
        Path(args.blind_pack).resolve(),
        Path(args.answer_key).resolve(),
        Path(args.decisions).resolve(),
        Path(args.output_json).resolve(),
        Path(args.output_md).resolve(),
        write_template=args.write_template,
    )
    print(json.dumps(payload, ensure_ascii=False, indent=2))
    raise SystemExit(0 if payload["ok"] else 2)


if __name__ == "__main__":
    main()