YAO
31ce04c655
Merge the beta-ready Yao Meta Skill architecture, report, evidence gate, and release-boundary updates.\n\nRelease boundary: beta/public testing is allowed; formal world-class, fully reviewed, or superiority claims remain blocked until the pending evidence gates are accepted.
279 lines
14 KiB
JSON
279 lines
14 KiB
JSON
{
|
|
"schema_version": "1.0",
|
|
"ok": true,
|
|
"summary": {
|
|
"pair_count": 5,
|
|
"judgment_count": 0,
|
|
"pending_count": 5,
|
|
"agreement_count": 0,
|
|
"disagreement_count": 0,
|
|
"invalid_decision_count": 0,
|
|
"answer_revealed_count": 0,
|
|
"pending_answer_hidden_count": 5,
|
|
"agreement_rate": null,
|
|
"needs_review": true,
|
|
"reviewer_metadata_present": false,
|
|
"reason_required": true,
|
|
"blind_review_attested": false,
|
|
"blind_review_completed_before_answer_key": false,
|
|
"answer_key_not_opened_before_decisions": false,
|
|
"raw_content_excluded_attested": true,
|
|
"reviewer_reason_required_attested": true,
|
|
"ready_for_human_evidence": false,
|
|
"failure_count": 0,
|
|
"reviewer_checklist_count": 5,
|
|
"reviewer_checklist_pending_count": 5,
|
|
"reviewer_checklist_invalid_count": 0,
|
|
"reviewer_checklist_ready_count": 0
|
|
},
|
|
"reviewer": "",
|
|
"reviewed_at": "",
|
|
"review_integrity": {
|
|
"blind_pack_sha256": "788604c67fa31178c9bba06a52731adcbee9c1e72c00767cf7950643812a6e5c",
|
|
"case_count": 5,
|
|
"case_ids": [
|
|
"skill-package-contract",
|
|
"output-eval-expectation",
|
|
"ir-before-packaging",
|
|
"near-neighbor-boundary",
|
|
"file-backed-governed-package"
|
|
],
|
|
"prompt_sha256_by_case": {
|
|
"skill-package-contract": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5",
|
|
"output-eval-expectation": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4",
|
|
"ir-before-packaging": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80",
|
|
"near-neighbor-boundary": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0",
|
|
"file-backed-governed-package": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d"
|
|
}
|
|
},
|
|
"reviewer_attestation": {
|
|
"blind_review_completed_before_answer_key": false,
|
|
"answer_key_not_opened_before_decisions": false,
|
|
"raw_content_excluded": true,
|
|
"reviewer_reason_required": true
|
|
},
|
|
"artifacts": {
|
|
"blind_pack": "reports/output_blind_review_pack.json",
|
|
"answer_key": "reports/output_blind_answer_key.json",
|
|
"decisions": "reports/output_review_decisions.json",
|
|
"json": "reports/output_review_adjudication.json",
|
|
"markdown": "reports/output_review_adjudication.md"
|
|
},
|
|
"template_written": false,
|
|
"pairs": [
|
|
{
|
|
"case_id": "skill-package-contract",
|
|
"status": "pending",
|
|
"expected_winner_variant": "",
|
|
"expected_revealed": false,
|
|
"reviewer_winner_variant": "",
|
|
"confidence": null,
|
|
"reason": "",
|
|
"prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5"
|
|
},
|
|
{
|
|
"case_id": "output-eval-expectation",
|
|
"status": "pending",
|
|
"expected_winner_variant": "",
|
|
"expected_revealed": false,
|
|
"reviewer_winner_variant": "",
|
|
"confidence": null,
|
|
"reason": "",
|
|
"prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4"
|
|
},
|
|
{
|
|
"case_id": "ir-before-packaging",
|
|
"status": "pending",
|
|
"expected_winner_variant": "",
|
|
"expected_revealed": false,
|
|
"reviewer_winner_variant": "",
|
|
"confidence": null,
|
|
"reason": "",
|
|
"prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80"
|
|
},
|
|
{
|
|
"case_id": "near-neighbor-boundary",
|
|
"status": "pending",
|
|
"expected_winner_variant": "",
|
|
"expected_revealed": false,
|
|
"reviewer_winner_variant": "",
|
|
"confidence": null,
|
|
"reason": "",
|
|
"prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0"
|
|
},
|
|
{
|
|
"case_id": "file-backed-governed-package",
|
|
"status": "pending",
|
|
"expected_winner_variant": "",
|
|
"expected_revealed": false,
|
|
"reviewer_winner_variant": "",
|
|
"confidence": null,
|
|
"reason": "",
|
|
"prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d"
|
|
}
|
|
],
|
|
"reviewer_checklist": [
|
|
{
|
|
"case_id": "skill-package-contract",
|
|
"readiness": "awaiting-decision",
|
|
"blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
|
|
"status": "pending",
|
|
"reviewer_winner_variant": "",
|
|
"answer_key_visible": false,
|
|
"prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5",
|
|
"blind_pack_path": "reports/output_blind_review_pack.json",
|
|
"decisions_path": "reports/output_review_decisions.json",
|
|
"commands": {
|
|
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
|
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
|
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
|
"adjudicate": "python3 scripts/yao.py output-review",
|
|
"refresh_review_studio": "python3 scripts/yao.py review-studio ."
|
|
},
|
|
"required_fields": {
|
|
"winner_variant": "A or B after reading only the blind review pack.",
|
|
"confidence": "Optional number from 0 to 1.",
|
|
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
|
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
|
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
|
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
|
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded."
|
|
},
|
|
"privacy_contract": [
|
|
"Do not paste raw private user data into the decision reason.",
|
|
"Do not open the answer key before reviewer choices are recorded.",
|
|
"Leave winner_variant blank when the reviewer is not ready to decide."
|
|
]
|
|
},
|
|
{
|
|
"case_id": "output-eval-expectation",
|
|
"readiness": "awaiting-decision",
|
|
"blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
|
|
"status": "pending",
|
|
"reviewer_winner_variant": "",
|
|
"answer_key_visible": false,
|
|
"prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4",
|
|
"blind_pack_path": "reports/output_blind_review_pack.json",
|
|
"decisions_path": "reports/output_review_decisions.json",
|
|
"commands": {
|
|
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
|
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
|
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
|
"adjudicate": "python3 scripts/yao.py output-review",
|
|
"refresh_review_studio": "python3 scripts/yao.py review-studio ."
|
|
},
|
|
"required_fields": {
|
|
"winner_variant": "A or B after reading only the blind review pack.",
|
|
"confidence": "Optional number from 0 to 1.",
|
|
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
|
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
|
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
|
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
|
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded."
|
|
},
|
|
"privacy_contract": [
|
|
"Do not paste raw private user data into the decision reason.",
|
|
"Do not open the answer key before reviewer choices are recorded.",
|
|
"Leave winner_variant blank when the reviewer is not ready to decide."
|
|
]
|
|
},
|
|
{
|
|
"case_id": "ir-before-packaging",
|
|
"readiness": "awaiting-decision",
|
|
"blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
|
|
"status": "pending",
|
|
"reviewer_winner_variant": "",
|
|
"answer_key_visible": false,
|
|
"prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80",
|
|
"blind_pack_path": "reports/output_blind_review_pack.json",
|
|
"decisions_path": "reports/output_review_decisions.json",
|
|
"commands": {
|
|
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
|
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
|
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
|
"adjudicate": "python3 scripts/yao.py output-review",
|
|
"refresh_review_studio": "python3 scripts/yao.py review-studio ."
|
|
},
|
|
"required_fields": {
|
|
"winner_variant": "A or B after reading only the blind review pack.",
|
|
"confidence": "Optional number from 0 to 1.",
|
|
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
|
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
|
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
|
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
|
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded."
|
|
},
|
|
"privacy_contract": [
|
|
"Do not paste raw private user data into the decision reason.",
|
|
"Do not open the answer key before reviewer choices are recorded.",
|
|
"Leave winner_variant blank when the reviewer is not ready to decide."
|
|
]
|
|
},
|
|
{
|
|
"case_id": "near-neighbor-boundary",
|
|
"readiness": "awaiting-decision",
|
|
"blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
|
|
"status": "pending",
|
|
"reviewer_winner_variant": "",
|
|
"answer_key_visible": false,
|
|
"prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0",
|
|
"blind_pack_path": "reports/output_blind_review_pack.json",
|
|
"decisions_path": "reports/output_review_decisions.json",
|
|
"commands": {
|
|
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
|
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
|
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
|
"adjudicate": "python3 scripts/yao.py output-review",
|
|
"refresh_review_studio": "python3 scripts/yao.py review-studio ."
|
|
},
|
|
"required_fields": {
|
|
"winner_variant": "A or B after reading only the blind review pack.",
|
|
"confidence": "Optional number from 0 to 1.",
|
|
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
|
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
|
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
|
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
|
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded."
|
|
},
|
|
"privacy_contract": [
|
|
"Do not paste raw private user data into the decision reason.",
|
|
"Do not open the answer key before reviewer choices are recorded.",
|
|
"Leave winner_variant blank when the reviewer is not ready to decide."
|
|
]
|
|
},
|
|
{
|
|
"case_id": "file-backed-governed-package",
|
|
"readiness": "awaiting-decision",
|
|
"blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.",
|
|
"status": "pending",
|
|
"reviewer_winner_variant": "",
|
|
"answer_key_visible": false,
|
|
"prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d",
|
|
"blind_pack_path": "reports/output_blind_review_pack.json",
|
|
"decisions_path": "reports/output_review_decisions.json",
|
|
"commands": {
|
|
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
|
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
|
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
|
"adjudicate": "python3 scripts/yao.py output-review",
|
|
"refresh_review_studio": "python3 scripts/yao.py review-studio ."
|
|
},
|
|
"required_fields": {
|
|
"winner_variant": "A or B after reading only the blind review pack.",
|
|
"confidence": "Optional number from 0 to 1.",
|
|
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
|
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
|
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
|
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
|
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded."
|
|
},
|
|
"privacy_contract": [
|
|
"Do not paste raw private user data into the decision reason.",
|
|
"Do not open the answer key before reviewer choices are recorded.",
|
|
"Leave winner_variant blank when the reviewer is not ready to decide."
|
|
]
|
|
}
|
|
],
|
|
"failures": []
|
|
}
|