From a943662be19a97c5ad32d292d65d2cc47f767ca0 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 23:50:18 +0800 Subject: [PATCH] Reject nested output review import leaks --- reports/context_budget.json | 12 +++---- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 6 ++-- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-os-2-review.md | 2 +- scripts/import_output_review_decisions.py | 18 +++++++--- tests/verify_output_review_adjudication.py | 38 ++++++++++++++++++++++ 8 files changed, 65 insertions(+), 17 deletions(-) diff --git a/reports/context_budget.json b/reports/context_budget.json index 73085ae..36e92ee 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1069909, + "other_text_tokens": 1070397, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1070706, - "deferred_resource_tokens": 495345, + "estimated_total_text_tokens": 1071194, + "deferred_resource_tokens": 495462, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435247, + "estimated_tokens": 435364, "file_count": 139 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435247, + "estimated_tokens": 435364, "file_count": 139 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435247, + "estimated_tokens": 435364, "file_count": 139, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index ffedf28..8e3ba33 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 495345 | `governed` | scripts:435247 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 495462 | `governed` | scripts:435364 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 4b51ff2..aba4d65 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 495345, + "deferred_resource_tokens": 495462, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435247, + "estimated_tokens": 435364, "file_count": 139 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435247, + "estimated_tokens": 435364, "file_count": 139, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 06bf013..fd4b2e6 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 230, - "package_sha256": "3335b896135939f1438e6bfe8b56dc050a1d95bcb1807986c9146f24725b97d6" + "package_sha256": "2bd5ae0684cf7508cce7611a6a840c287d35067f13e5f78f33c45667c501280c" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 8e30e79..56f33c1 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `230` -- Package SHA256: `3335b896135939f1438e6bfe8b56dc050a1d95bcb1807986c9146f24725b97d6` +- Package SHA256: `2bd5ae0684cf7508cce7611a6a840c287d35067f13e5f78f33c45667c501280c` ## Failures diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 5a8aa75..9591ef6 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -52,7 +52,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Review Studio now avoids over-claiming release readiness when blind A/B adjudication, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, three warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion. - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. - Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. -- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, rejects raw prompt/output/transcript/message and answer-key fields, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist. +- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, recursively rejects raw prompt/output/transcript/message and answer-key fields, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. - Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes. diff --git a/scripts/import_output_review_decisions.py b/scripts/import_output_review_decisions.py index 2ab6977..8943076 100644 --- a/scripts/import_output_review_decisions.py +++ b/scripts/import_output_review_decisions.py @@ -142,9 +142,19 @@ def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None return round(parsed, 3), None -def forbidden_fields(item: dict[str, Any]) -> list[str]: - keys = {str(key) for key in item} - return sorted((keys & RAW_CONTENT_FIELDS) | (keys & ANSWER_KEY_FIELDS)) +def forbidden_field_paths(value: Any, prefix: str) -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key).strip() + child_path = f"{prefix}.{key_text}" + if key_text in RAW_CONTENT_FIELDS or key_text in ANSWER_KEY_FIELDS: + found.append(child_path) + found.extend(forbidden_field_paths(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + found.extend(forbidden_field_paths(child, f"{prefix}[{index}]")) + return found def normalize_decisions( @@ -158,7 +168,7 @@ def normalize_decisions( if not isinstance(item, dict): failures.append(f"decision #{index} must be an object") continue - blocked_fields = forbidden_fields(item) + blocked_fields = forbidden_field_paths(item, f"decision #{index}") if blocked_fields: failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}") case_id = str(item.get("case_id", "")).strip() diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py index b9713dc..f6afcd2 100644 --- a/tests/verify_output_review_adjudication.py +++ b/tests/verify_output_review_adjudication.py @@ -314,6 +314,44 @@ def main() -> None: assert any("forbidden raw or answer-key fields" in failure for failure in private_payload["failures"]), private_payload assert not (tmp_root / "private_decisions.json").exists(), private_payload + nested_private_source = tmp_root / "nested_private_source.json" + nested_private_source.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "A", + "metadata": {"raw_output": "nested raw output must not be imported"}, + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + nested_private_proc = run( + [ + str(IMPORTER), + "--input", + str(nested_private_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "nested_private_decisions.json"), + ], + check=False, + ) + nested_private_payload = json.loads(nested_private_proc.stdout) + assert nested_private_proc.returncode == 2, nested_private_payload + assert nested_private_payload["ok"] is False, nested_private_payload + assert any("decision #1.metadata.raw_output" in failure for failure in nested_private_payload["failures"]), nested_private_payload + assert not (tmp_root / "nested_private_decisions.json").exists(), nested_private_payload + unknown_case = tmp_root / "unknown_case.json" unknown_case.write_text( json.dumps(