From 1945775f93f9a449029b1ffebd548ededc2d9cc2 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Wed, 17 Jun 2026 00:07:59 +0800 Subject: [PATCH] Share output review privacy guard --- reports/architecture_maintainability.json | 8 +-- reports/architecture_maintainability.md | 8 +-- reports/context_budget.json | 20 +++---- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 12 ++-- reports/python_compatibility.json | 10 +++- reports/python_compatibility.md | 4 +- reports/security_trust_report.json | 30 ++++++++-- reports/security_trust_report.md | 11 ++-- reports/skill-os-2-review.md | 7 ++- scripts/import_output_review_decisions.py | 62 +------------------- scripts/output_review_privacy.py | 66 ++++++++++++++++++++++ scripts/world_class_human_evidence.py | 59 +------------------ tests/verify_output_review_adjudication.py | 6 ++ 14 files changed, 145 insertions(+), 160 deletions(-) create mode 100644 scripts/output_review_privacy.py diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index bbe1faac..79270973 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,13 +1,13 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { - "python_file_count": 214, - "script_file_count": 139, + "python_file_count": 215, + "script_file_count": 140, "test_file_count": 75, - "internal_module_count": 55, + "internal_module_count": 56, "cli_script_count": 86, "command_handler_count": 68, "entrypoint_command_handler_count": 18, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index e4f40ed1..7d12ee08 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,14 +1,14 @@ # Architecture Maintainability -Generated at: `2026-06-16` +Generated at: `2026-06-17` ## Summary - decision: `pass` -- python files: `214` -- scripts: `139` +- python files: `215` +- scripts: `140` - tests: `75` -- internal modules: `55` +- internal modules: `56` - CLI scripts: `86` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` diff --git a/reports/context_budget.json b/reports/context_budget.json index be887e6d..2a532001 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1070976, + "other_text_tokens": 1071125, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1071773, - "deferred_resource_tokens": 495581, + "estimated_total_text_tokens": 1071922, + "deferred_resource_tokens": 495322, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435483, - "file_count": 139 + "estimated_tokens": 435224, + "file_count": 140 }, { "path": "references", @@ -36,8 +36,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435483, - "file_count": 139 + "estimated_tokens": 435224, + "file_count": 140 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435483, - "file_count": 139, + "estimated_tokens": 435224, + "file_count": 140, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 675, + "relevant_file_count": 677, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index d7ef9fdd..b48f7da9 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 495581 | `governed` | scripts:435483 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 495322 | `governed` | scripts:435224 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index fdad6eb7..bceb0c86 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 495581, + "deferred_resource_tokens": 495322, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435483, - "file_count": 139 + "estimated_tokens": 435224, + "file_count": 140 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435483, - "file_count": 139, + "estimated_tokens": 435224, + "file_count": 140, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 282d30f8..1c479960 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,11 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-17", "root": ".", "summary": { "target_python": "3.11", - "file_count": 217, + "file_count": 218, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -244,6 +244,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/output_review_privacy.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/prepare_output_review_kit.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index b03b8135..03a8bf61 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,12 +1,12 @@ # Python Compatibility -Generated at: `2026-06-16` +Generated at: `2026-06-17` ## Summary - decision: `pass` - target python: `3.11` -- files scanned: `217` +- files scanned: `218` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index a6fb661c..3284e101 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 230, - "script_count": 139, - "internal_module_count": 53, + "scanned_files": 231, + "script_count": 140, + "internal_module_count": 54, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 230, - "package_sha256": "e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16" + "package_hash_file_count": 231, + "package_sha256": "fcfe5f3c722285daf14b6f83118ad71bdca21694d86d870d234d9f5df2f4e627" }, "failures": [], "warnings": [], @@ -514,6 +514,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/output_review_privacy.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/prepare_output_review_kit.py", "interface": "cli", @@ -2027,7 +2041,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 53, + "skipped_count": 54, "failed_scripts": [], "results": [ { @@ -2936,6 +2950,10 @@ "path": "scripts/html_rendering.py", "reason": "internal module" }, + { + "path": "scripts/output_review_privacy.py", + "reason": "internal module" + }, { "path": "scripts/review_studio_action_evidence.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 85f8c96f..ea64f0a1 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `230` -- Scripts: `139` -- Internal script modules: `53` +- Scanned files: `231` +- Scripts: `140` +- Internal script modules: `54` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `230` -- Package SHA256: `e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16` +- Package hash files: `231` +- Package SHA256: `fcfe5f3c722285daf14b6f83118ad71bdca21694d86d870d234d9f5df2f4e627` ## Failures @@ -96,6 +96,7 @@ | scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/output_review_privacy.py | internal-module | True | False | False | False | False | False | False | Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively. | | scripts/prepare_output_review_kit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/prepare_world_class_submission_kit.py | cli | True | True | True | False | False | True | False | Prepares editable world-class evidence intake packets without counting drafts as accepted evidence. | | scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 73225a52..5e26c771 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -22,6 +22,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. - World-Class Human Evidence Guard v0 so human-adjudication decision artifacts recursively reject raw content, credential, secret, token, and answer-key fields before blind A/B review evidence can be accepted. +- Output Review Privacy Guard v0 so the blind-review importer and world-class human evidence validator share one recursive blocked-field contract for raw content, credential, secret, token, and answer-key leakage. - World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance. - World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free. - World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture. @@ -64,13 +65,13 @@ This is still not the final world-class state. Target-native behavior contracts | --- | --- | --- | | Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed | | Target Compiler | `scripts/compile_skill.py`, `scripts/compile_skill_targets.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | -| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | +| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/output_review_privacy.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | | Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed | | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | -| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_human_evidence.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, recursive human decision privacy and answer-key validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_human_evidence.py`, `scripts/output_review_privacy.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, recursive human decision privacy and answer-key validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed | | World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | @@ -131,7 +132,7 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `53` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `139` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `54` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `140` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | diff --git a/scripts/import_output_review_decisions.py b/scripts/import_output_review_decisions.py index de45d10a..068be25e 100644 --- a/scripts/import_output_review_decisions.py +++ b/scripts/import_output_review_decisions.py @@ -17,55 +17,13 @@ from adjudicate_output_review import ( normalize_variant, pair_index, ) +from output_review_privacy import forbidden_decision_field_paths ROOT = Path(__file__).resolve().parent.parent SCRIPT_INTERFACE = "cli" SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file." -RAW_CONTENT_FIELDS = { - "api_key", - "prompt", - "prompts", - "input", - "inputs", - "output", - "outputs", - "transcript", - "transcripts", - "message", - "messages", - "user_message", - "user_messages", - "assistant_message", - "assistant_messages", - "model_output", - "baseline_output", - "with_skill_output", - "raw_content", - "raw_prompt", - "raw_provider_prompt", - "raw_user_content", - "raw_output", - "credential", - "credentials", - "secret", - "secrets", - "token", -} - -ANSWER_KEY_FIELDS = { - "expected", - "expected_winner", - "expected_winner_role", - "expected_winner_variant", - "answer_key", - "label", - "variant_label", - "baseline_label", - "with_skill_label", -} - def display_path(path: Path) -> str: try: @@ -153,22 +111,6 @@ def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None return round(parsed, 3), None -def forbidden_field_paths(value: Any, prefix: str) -> list[str]: - found: list[str] = [] - if isinstance(value, dict): - for key, child in value.items(): - key_text = str(key).strip() - child_path = f"{prefix}.{key_text}" - normalized_key = key_text.casefold() - if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS: - found.append(child_path) - found.extend(forbidden_field_paths(child, child_path)) - elif isinstance(value, list): - for index, child in enumerate(value): - found.extend(forbidden_field_paths(child, f"{prefix}[{index}]")) - return found - - def normalize_decisions( source_items: list[dict[str, Any]], case_ids: set[str], @@ -180,7 +122,7 @@ def normalize_decisions( if not isinstance(item, dict): failures.append(f"decision #{index} must be an object") continue - blocked_fields = forbidden_field_paths(item, f"decision #{index}") + blocked_fields = forbidden_decision_field_paths(item, f"decision #{index}") if blocked_fields: failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}") case_id = str(item.get("case_id", "")).strip() diff --git a/scripts/output_review_privacy.py b/scripts/output_review_privacy.py new file mode 100644 index 00000000..b940d5ca --- /dev/null +++ b/scripts/output_review_privacy.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively." + +RAW_CONTENT_FIELDS = { + "api_key", + "assistant_message", + "assistant_messages", + "baseline_output", + "credential", + "credentials", + "input", + "inputs", + "message", + "messages", + "model_output", + "output", + "outputs", + "prompt", + "prompts", + "raw_content", + "raw_output", + "raw_prompt", + "raw_provider_prompt", + "raw_user_content", + "secret", + "secrets", + "token", + "transcript", + "transcripts", + "user_message", + "user_messages", + "with_skill_output", +} + +ANSWER_KEY_FIELDS = { + "answer_key", + "baseline_label", + "expected", + "expected_winner", + "expected_winner_role", + "expected_winner_variant", + "label", + "variant_label", + "with_skill_label", +} + +BLOCKED_DECISION_FIELDS = RAW_CONTENT_FIELDS | ANSWER_KEY_FIELDS + + +def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key).strip() + child_path = f"{prefix}.{key_text}" + if key_text.casefold() in BLOCKED_DECISION_FIELDS: + found.append(child_path) + found.extend(forbidden_decision_field_paths(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]")) + return found diff --git a/scripts/world_class_human_evidence.py b/scripts/world_class_human_evidence.py index 90e9573d..a6fc0dfd 100644 --- a/scripts/world_class_human_evidence.py +++ b/scripts/world_class_human_evidence.py @@ -2,52 +2,13 @@ import re from typing import Any +from output_review_privacy import forbidden_decision_field_paths + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows." REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$") -RAW_CONTENT_FIELDS = { - "api_key", - "assistant_message", - "assistant_messages", - "baseline_output", - "credential", - "credentials", - "input", - "inputs", - "message", - "messages", - "model_output", - "output", - "outputs", - "prompt", - "prompts", - "raw_content", - "raw_output", - "raw_prompt", - "raw_provider_prompt", - "raw_user_content", - "secret", - "secrets", - "token", - "transcript", - "transcripts", - "user_message", - "user_messages", - "with_skill_output", -} -ANSWER_KEY_FIELDS = { - "answer_key", - "baseline_label", - "expected", - "expected_winner", - "expected_winner_role", - "expected_winner_variant", - "label", - "variant_label", - "with_skill_label", -} def add_error(errors: list[str], condition: bool, message: str) -> None: @@ -81,22 +42,6 @@ def duplicate_case_ids(ids: list[str]) -> list[str]: return duplicates -def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]: - found: list[str] = [] - if isinstance(value, dict): - for key, child in value.items(): - key_text = str(key).strip() - child_path = f"{prefix}.{key_text}" - normalized_key = key_text.casefold() - if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS: - found.append(child_path) - found.extend(forbidden_decision_field_paths(child, child_path)) - elif isinstance(value, list): - for index, child in enumerate(value): - found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]")) - return found - - def confidence_valid(value: Any) -> bool: if value is None or value == "": return True diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py index 615a448d..5987f349 100644 --- a/tests/verify_output_review_adjudication.py +++ b/tests/verify_output_review_adjudication.py @@ -7,6 +7,10 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from output_review_privacy import BLOCKED_DECISION_FIELDS # noqa: E402 + OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" ADJUDICATOR = ROOT / "scripts" / "adjudicate_output_review.py" IMPORTER = ROOT / "scripts" / "import_output_review_decisions.py" @@ -24,6 +28,8 @@ def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str] def main() -> None: + assert {"api_key", "raw_provider_prompt", "expected_winner_role"} <= BLOCKED_DECISION_FIELDS + tmp_root = ROOT / "tests" / "tmp_output_review_adjudication" if tmp_root.exists(): shutil.rmtree(tmp_root)