yao-meta-skill/reports/output_review_adjudication.json

{
  "schema_version": "1.0",
  "ok": true,
  "summary": {
    "pair_count": 5,
    "judgment_count": 0,
    "pending_count": 5,
    "agreement_count": 0,
    "disagreement_count": 0,
    "invalid_decision_count": 0,
    "answer_revealed_count": 0,
    "pending_answer_hidden_count": 5,
    "agreement_rate": null,
    "needs_review": true,
    "failure_count": 0,
    "reviewer_checklist_count": 5,
    "reviewer_checklist_pending_count": 5,
    "reviewer_checklist_invalid_count": 0,
    "reviewer_checklist_ready_count": 0
  },
  "reviewer": "",
  "reviewed_at": "",
  "artifacts": {
    "blind_pack": "reports/output_blind_review_pack.json",
    "answer_key": "reports/output_blind_answer_key.json",
    "decisions": "reports/output_review_decisions.json",
    "json": "reports/output_review_adjudication.json",
    "markdown": "reports/output_review_adjudication.md"
  },
  "template_written": false,
  "pairs": [
    {
      "case_id": "skill-package-contract",
      "status": "pending",
      "expected_winner_variant": "",
      "expected_revealed": false,
      "reviewer_winner_variant": "",
      "confidence": null,
      "reason": "",
      "prompt": "Turn this repeated workflow into a reusable team skill package."
    },
    {
      "case_id": "output-eval-expectation",
      "status": "pending",
      "expected_winner_variant": "",
      "expected_revealed": false,
      "reviewer_winner_variant": "",
      "confidence": null,
      "reason": "",
      "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline."
    },
    {
      "case_id": "ir-before-packaging",
      "status": "pending",
      "expected_winner_variant": "",
      "expected_revealed": false,
      "reviewer_winner_variant": "",
      "confidence": null,
      "reason": "",
      "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets."
    },
    {
      "case_id": "near-neighbor-boundary",
      "status": "pending",
      "expected_winner_variant": "",
      "expected_revealed": false,
      "reviewer_winner_variant": "",
      "confidence": null,
      "reason": "",
      "prompt": "I only need a one-off summary of these notes, not a reusable process."
    },
    {
      "case_id": "file-backed-governed-package",
      "status": "pending",
      "expected_winner_variant": "",
      "expected_revealed": false,
      "reviewer_winner_variant": "",
      "confidence": null,
      "reason": "",
      "prompt": "Turn the attached release brief source into a governed skill package."
    }
  ],
  "reviewer_checklist": [
    {
      "case_id": "skill-package-contract",
      "readiness": "awaiting-decision",
      "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
      "status": "pending",
      "reviewer_winner_variant": "",
      "answer_key_visible": false,
      "prompt": "Turn this repeated workflow into a reusable team skill package.",
      "blind_pack_path": "reports/output_blind_review_pack.json",
      "decisions_path": "reports/output_review_decisions.json",
      "commands": {
        "prepare_review_kit": "python3 scripts/yao.py output-review-kit",
        "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
        "adjudicate": "python3 scripts/yao.py output-review",
        "refresh_review_studio": "python3 scripts/yao.py review-studio ."
      },
      "required_fields": {
        "winner_variant": "A or B after reading only the blind review pack.",
        "confidence": "Optional number from 0 to 1.",
        "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication."
      },
      "privacy_contract": [
        "Do not paste raw private user data into the decision reason.",
        "Do not open the answer key before reviewer choices are recorded.",
        "Leave winner_variant blank when the reviewer is not ready to decide."
      ]
    },
    {
      "case_id": "output-eval-expectation",
      "readiness": "awaiting-decision",
      "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
      "status": "pending",
      "reviewer_winner_variant": "",
      "answer_key_visible": false,
      "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.",
      "blind_pack_path": "reports/output_blind_review_pack.json",
      "decisions_path": "reports/output_review_decisions.json",
      "commands": {
        "prepare_review_kit": "python3 scripts/yao.py output-review-kit",
        "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
        "adjudicate": "python3 scripts/yao.py output-review",
        "refresh_review_studio": "python3 scripts/yao.py review-studio ."
      },
      "required_fields": {
        "winner_variant": "A or B after reading only the blind review pack.",
        "confidence": "Optional number from 0 to 1.",
        "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication."
      },
      "privacy_contract": [
        "Do not paste raw private user data into the decision reason.",
        "Do not open the answer key before reviewer choices are recorded.",
        "Leave winner_variant blank when the reviewer is not ready to decide."
      ]
    },
    {
      "case_id": "ir-before-packaging",
      "readiness": "awaiting-decision",
      "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
      "status": "pending",
      "reviewer_winner_variant": "",
      "answer_key_visible": false,
      "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.",
      "blind_pack_path": "reports/output_blind_review_pack.json",
      "decisions_path": "reports/output_review_decisions.json",
      "commands": {
        "prepare_review_kit": "python3 scripts/yao.py output-review-kit",
        "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
        "adjudicate": "python3 scripts/yao.py output-review",
        "refresh_review_studio": "python3 scripts/yao.py review-studio ."
      },
      "required_fields": {
        "winner_variant": "A or B after reading only the blind review pack.",
        "confidence": "Optional number from 0 to 1.",
        "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication."
      },
      "privacy_contract": [
        "Do not paste raw private user data into the decision reason.",
        "Do not open the answer key before reviewer choices are recorded.",
        "Leave winner_variant blank when the reviewer is not ready to decide."
      ]
    },
    {
      "case_id": "near-neighbor-boundary",
      "readiness": "awaiting-decision",
      "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
      "status": "pending",
      "reviewer_winner_variant": "",
      "answer_key_visible": false,
      "prompt": "I only need a one-off summary of these notes, not a reusable process.",
      "blind_pack_path": "reports/output_blind_review_pack.json",
      "decisions_path": "reports/output_review_decisions.json",
      "commands": {
        "prepare_review_kit": "python3 scripts/yao.py output-review-kit",
        "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
        "adjudicate": "python3 scripts/yao.py output-review",
        "refresh_review_studio": "python3 scripts/yao.py review-studio ."
      },
      "required_fields": {
        "winner_variant": "A or B after reading only the blind review pack.",
        "confidence": "Optional number from 0 to 1.",
        "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication."
      },
      "privacy_contract": [
        "Do not paste raw private user data into the decision reason.",
        "Do not open the answer key before reviewer choices are recorded.",
        "Leave winner_variant blank when the reviewer is not ready to decide."
      ]
    },
    {
      "case_id": "file-backed-governed-package",
      "readiness": "awaiting-decision",
      "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
      "status": "pending",
      "reviewer_winner_variant": "",
      "answer_key_visible": false,
      "prompt": "Turn the attached release brief source into a governed skill package.",
      "blind_pack_path": "reports/output_blind_review_pack.json",
      "decisions_path": "reports/output_review_decisions.json",
      "commands": {
        "prepare_review_kit": "python3 scripts/yao.py output-review-kit",
        "write_template": "python3 scripts/adjudicate_output_review.py --write-template",
        "adjudicate": "python3 scripts/yao.py output-review",
        "refresh_review_studio": "python3 scripts/yao.py review-studio ."
      },
      "required_fields": {
        "winner_variant": "A or B after reading only the blind review pack.",
        "confidence": "Optional number from 0 to 1.",
        "reason": "Short rationale; do not reveal baseline or with-skill labels before adjudication."
      },
      "privacy_contract": [
        "Do not paste raw private user data into the decision reason.",
        "Do not open the answer key before reviewer choices are recorded.",
        "Leave winner_variant blank when the reviewer is not ready to decide."
      ]
    }
  ],
  "failures": []
}