feat: add output review checklist

scripts/adjudicate_output_review.py

@@ -216,6 +216,96 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str,
     }
 
 
+def checklist_readiness(pair: dict[str, Any]) -> tuple[str, str]:
+    status = pair.get("status")
+    if status in {"match", "disagree"}:
+        return "adjudicated", "Reviewer decision is valid; answer key is revealed for this case."
+    if status == "invalid":
+        return "fix-decision", "Reviewer decision exists but failed validation; answer key remains hidden."
+    return "awaiting-decision", "Reviewer has not selected A or B yet; answer key remains hidden."
+
+
+def build_reviewer_checklist(
+    pairs: list[dict[str, Any]],
+    blind_pack_path: Path,
+    decisions_path: Path,
+) -> list[dict[str, Any]]:
+    checklist = []
+    for pair in pairs:
+        readiness, blocking_reason = checklist_readiness(pair)
+        checklist.append(
+            {
+                "case_id": pair.get("case_id", ""),
+                "readiness": readiness,
+                "blocking_reason": blocking_reason,
+                "status": pair.get("status", "pending"),
+                "reviewer_winner_variant": pair.get("reviewer_winner_variant", ""),
+                "answer_key_visible": bool(pair.get("expected_revealed")),
+                "prompt": pair.get("prompt", ""),
+                "blind_pack_path": display_path(blind_pack_path),
+                "decisions_path": display_path(decisions_path),
+                "commands": {
+                    "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
+                    "adjudicate": "python3 scripts/yao.py output-review",
+                    "refresh_review_studio": "python3 scripts/yao.py review-studio .",
+                },
+                "required_fields": {
+                    "winner_variant": "A or B after reading only the blind review pack.",
+                    "confidence": "Optional number from 0 to 1.",
+                    "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication.",
+                },
+                "privacy_contract": [
+                    "Do not paste raw private user data into the decision reason.",
+                    "Do not open the answer key before reviewer choices are recorded.",
+                    "Leave winner_variant blank when the reviewer is not ready to decide.",
+                ],
+            }
+        )
+    return checklist
+
+
+def add_checklist_summary(summary: dict[str, Any], checklist: list[dict[str, Any]]) -> dict[str, Any]:
+    enriched = dict(summary)
+    enriched["reviewer_checklist_count"] = len(checklist)
+    enriched["reviewer_checklist_pending_count"] = sum(1 for item in checklist if item["readiness"] == "awaiting-decision")
+    enriched["reviewer_checklist_invalid_count"] = sum(1 for item in checklist if item["readiness"] == "fix-decision")
+    enriched["reviewer_checklist_ready_count"] = sum(1 for item in checklist if item["readiness"] == "adjudicated")
+    return enriched
+
+
+def render_reviewer_checklist(checklist: list[dict[str, Any]]) -> list[str]:
+    lines = [
+        "## Reviewer Checklist",
+        "",
+        "| Case | Readiness | Answer key | Decision file |",
+        "| --- | --- | --- | --- |",
+    ]
+    if not checklist:
+        lines.append("| `none` | `adjudicated` | n/a | none |")
+        return lines
+    for item in checklist:
+        answer_key = "visible" if item.get("answer_key_visible") else "hidden"
+        lines.append(
+            f"| `{item['case_id']}` | `{item['readiness']}` | `{answer_key}` | `{item['decisions_path']}` |"
+        )
+    for item in checklist:
+        lines.extend(["", f"### {item['case_id']}", ""])
+        lines.append(f"- readiness: `{item['readiness']}`")
+        lines.append(f"- blocking reason: {item['blocking_reason']}")
+        lines.append(f"- answer key visible: `{str(item['answer_key_visible']).lower()}`")
+        lines.append(f"- blind pack: `{item['blind_pack_path']}`")
+        lines.append(f"- decisions: `{item['decisions_path']}`")
+        lines.extend(["", "#### Commands", ""])
+        for label, command in item.get("commands", {}).items():
+            lines.append(f"- {label}: `{command}`")
+        lines.extend(["", "#### Required Fields", ""])
+        for label, description in item.get("required_fields", {}).items():
+            lines.append(f"- {label}: {description}")
+        lines.extend(["", "#### Privacy Contract", ""])
+        lines.extend(f"- {contract}" for contract in item.get("privacy_contract", []))
+    return lines
+
+
 def render_markdown(payload: dict[str, Any]) -> str:
     summary = payload["summary"]
     lines = [
@@ -230,6 +320,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
         f"- Invalid decisions: `{summary['invalid_decision_count']}`",
         f"- Answer keys revealed: `{summary['answer_revealed_count']}`",
         f"- Pending/invalid answers hidden: `{summary['pending_answer_hidden_count']}`",
+        f"- Reviewer checklist: `{summary['reviewer_checklist_ready_count']}` ready / `{summary['reviewer_checklist_count']}` total",
         "",
     ]
     if summary["judgment_count"] == 0:
@@ -262,6 +353,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
         lines.extend(["", "## Failures", ""])
         for failure in payload["failures"]:
             lines.append(f"- {failure}")
+    lines.extend(["", *render_reviewer_checklist(payload.get("reviewer_checklist", []))])
     lines.extend(
         [
             "",
@@ -314,6 +406,8 @@ def adjudicate_output_review(
         failures.extend(pair_failures)
 
     summary = build_summary(adjudicated_pairs, failures)
+    reviewer_checklist = build_reviewer_checklist(adjudicated_pairs, blind_pack_path, decisions_path)
+    summary = add_checklist_summary(summary, reviewer_checklist)
     payload = {
         "schema_version": "1.0",
         "ok": not failures,
@@ -329,6 +423,7 @@ def adjudicate_output_review(
         },
         "template_written": template_written,
         "pairs": adjudicated_pairs,
+        "reviewer_checklist": reviewer_checklist,
         "failures": failures,
     }
     output_json.parent.mkdir(parents=True, exist_ok=True)

scripts/render_review_studio.py

@@ -10,13 +10,13 @@ from review_studio_data import evidence_paths, insight_cards, load_review_data
 from review_studio_formatting import registry_package_summary, render_kv_grid
 from review_studio_gates import add_blockers_from_gate, build_gates, status_label, weighted_score
 from review_studio_layout import render_review_nav, review_studio_css
+from review_studio_output_review import render_output_review_section
 from review_studio_world_class import render_world_class_evidence_entries, render_world_class_intake_checklist
 
 
 ROOT = Path(__file__).resolve().parent.parent
 
 
-
 def display_path(skill_dir: Path, path: Path) -> str:
     try:
         return str(path.resolve().relative_to(skill_dir.resolve()))
@@ -514,9 +514,12 @@ def render_html(report: dict[str, Any]) -> str:
             "invalid_decision_count",
             "answer_revealed_count",
             "pending_answer_hidden_count",
+            "reviewer_checklist_count",
+            "reviewer_checklist_pending_count",
         ],
         "review adjudication report missing",
     )
+    output_review_section_html = render_output_review_section(report["data"].get("output_review_adjudication", {}))
     blueprint_panel = render_kv_grid(
         blueprint_summary,
         [
@@ -723,6 +726,7 @@ def render_html(report: dict[str, Any]) -> str:
       <div class="panel"><h2>审定报告</h2>{review_panel}</div>
     </section>
 
+    {output_review_section_html}
     <section class="twocol">
       <div class="panel"><h2>发布标准</h2><p>Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。</p></div>
       <div class="panel"><h2>人工结论</h2><p>没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。</p></div>

scripts/review_studio_layout.py

@@ -253,6 +253,77 @@ def review_studio_css() -> str:
       margin: 0;
       padding-left: 18px;
     }
+    .output-review-grid {
+      display: grid;
+      grid-template-columns: minmax(0, 1fr);
+      gap: 16px;
+      margin-top: 16px;
+    }
+    .output-review-card {
+      border: 1px solid var(--line);
+      border-radius: 8px;
+      padding: 18px;
+      background: #fff;
+      display: grid;
+      gap: 12px;
+      min-width: 0;
+    }
+    .output-review-card.awaiting-decision,
+    .output-review-card.fix-decision { border-left: 4px solid var(--warn); }
+    .output-review-card.adjudicated { border-left: 4px solid var(--pass); }
+    .output-review-card span,
+    .output-review-card p,
+    .output-review-card dd,
+    .output-review-card li {
+      color: var(--muted);
+      font-size: 14px;
+      overflow-wrap: anywhere;
+    }
+    .output-review-card dl {
+      display: grid;
+      grid-template-columns: 86px minmax(0, 1fr);
+      gap: 6px 10px;
+      margin: 0;
+    }
+    .output-review-card dt { color: var(--ink); font-size: 14px; }
+    .output-review-card dd { margin: 0; min-width: 0; }
+    .output-review-steps {
+      display: grid;
+      grid-template-columns: repeat(3, minmax(0, 1fr));
+      gap: 12px;
+      border-top: 1px solid var(--line);
+      padding-top: 12px;
+    }
+    .output-review-steps h4 {
+      margin: 0 0 6px;
+      color: var(--ink);
+      font-size: 14px;
+      font-weight: 500;
+    }
+    .output-review-steps ul {
+      margin: 0;
+      padding-left: 18px;
+    }
+    .output-review-commands,
+    .output-review-fields {
+      list-style: none;
+      padding: 0;
+      display: grid;
+      gap: 8px;
+    }
+    .output-review-commands li,
+    .output-review-fields li {
+      display: grid;
+      gap: 2px;
+      min-width: 0;
+      padding: 8px 0 0;
+      border-top: 1px solid var(--line);
+    }
+    .output-review-commands code {
+      overflow-wrap: anywhere;
+      white-space: normal;
+      display: block;
+    }
     .world-intake-grid {
       display: grid;
       grid-template-columns: minmax(0, 1fr);
@@ -373,7 +444,7 @@ def review_studio_css() -> str:
       font-size: 13px;
     }
     @media (max-width: 980px) {
-      .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .world-evidence-grid, .world-evidence-columns, .world-intake-grid, .world-intake-steps, .kv-grid { grid-template-columns: 1fr; }
+      .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-intake-grid, .world-intake-steps, .kv-grid { grid-template-columns: 1fr; }
       main { padding: 32px 18px 60px; }
       nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }
       nav a { flex: 0 0 auto; }

scripts/review_studio_output_review.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Output review HTML helpers for Review Studio."""
+
+import html
+from typing import Any
+
+
+SCRIPT_INTERFACE = "internal-module"
+SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render output review checklist cards."
+
+
+def render_required_fields(fields: dict[str, Any]) -> str:
+    if not fields:
+        return "<p class='muted'>暂无字段要求。</p>"
+    items = []
+    for key, value in fields.items():
+        items.append(f"<li><strong>{html.escape(str(key))}</strong><span>{html.escape(str(value))}</span></li>")
+    return "<ul class='output-review-fields'>" + "".join(items) + "</ul>"
+
+
+def render_command_list(commands: dict[str, Any]) -> str:
+    if not commands:
+        return "<p class='muted'>暂无命令。</p>"
+    items = []
+    for label, command in commands.items():
+        if not command:
+            continue
+        items.append(
+            "<li>"
+            f"<span>{html.escape(str(label))}</span>"
+            f"<code>{html.escape(str(command))}</code>"
+            "</li>"
+        )
+    return "<ul class='output-review-commands'>" + "".join(items) + "</ul>" if items else "<p class='muted'>暂无命令。</p>"
+
+
+def render_contract(items: list[Any]) -> str:
+    if not items:
+        return "<p class='muted'>暂无边界。</p>"
+    return "<ul>" + "".join(f"<li>{html.escape(str(item))}</li>" for item in items) + "</ul>"
+
+
+def render_output_review_checklist(adjudication: dict[str, Any]) -> str:
+    checklist = adjudication.get("reviewer_checklist", []) if isinstance(adjudication, dict) else []
+    if not checklist:
+        return "<p class='muted'>当前没有输出评审操作清单。</p>"
+    cards = []
+    for item in checklist:
+        readiness = str(item.get("readiness", "awaiting-decision"))
+        answer_key = "可见" if item.get("answer_key_visible") else "隐藏"
+        cards.append(
+            "<article class='output-review-card "
+            + html.escape(readiness)
+            + "'>"
+            f"<div><span>{html.escape(readiness)} · 答案{html.escape(answer_key)}</span>"
+            f"<h3>{html.escape(str(item.get('case_id', 'case')))}</h3></div>"
+            f"<p>{html.escape(str(item.get('blocking_reason', '')))}</p>"
+            f"<dl><dt>盲评包</dt><dd><code>{html.escape(str(item.get('blind_pack_path', '')))}</code></dd>"
+            f"<dt>决策表</dt><dd><code>{html.escape(str(item.get('decisions_path', '')))}</code></dd>"
+            f"<dt>提示词</dt><dd>{html.escape(str(item.get('prompt', '')))}</dd></dl>"
+            "<div class='output-review-steps'>"
+            "<div><h4>操作命令</h4>"
+            + render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {})
+            + "</div>"
+            "<div><h4>字段要求</h4>"
+            + render_required_fields(item.get("required_fields", {}) if isinstance(item.get("required_fields", {}), dict) else {})
+            + "</div>"
+            "<div><h4>隐私边界</h4>"
+            + render_contract(item.get("privacy_contract", []) if isinstance(item.get("privacy_contract", []), list) else [])
+            + "</div>"
+            "</div>"
+            "</article>"
+        )
+    return "<div class='output-review-grid'>" + "".join(cards) + "</div>"
+
+
+def render_output_review_section(adjudication: dict[str, Any]) -> str:
+    return (
+        "<section>"
+        "<h2>评审清单</h2>"
+        "<p class='muted'>每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。</p>"
+        f"{render_output_review_checklist(adjudication)}"
+        "</section>"
+    )