diff --git a/scripts/world_class_native_permission_evidence.py b/scripts/world_class_native_permission_evidence.py index c64de04..f6ca6e3 100644 --- a/scripts/world_class_native_permission_evidence.py +++ b/scripts/world_class_native_permission_evidence.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 from typing import Any +from output_review_privacy import forbidden_decision_field_paths + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows." @@ -23,6 +25,16 @@ def sorted_strings(value: Any) -> list[str]: return sorted(str(item) for item in value) if isinstance(value, list) else [] +def validate_no_raw_fields(payload: dict[str, Any], label: str, errors: list[str]) -> None: + blocked = forbidden_decision_field_paths(payload, label) + add_error( + errors, + not blocked, + f"native-permission-enforcement {label} must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked[:8]), + ) + + def validate_target_rows(probes: dict[str, Any], summary: dict[str, Any], errors: list[str]) -> None: targets = object_list(probes.get("targets", [])) target_count = real_int(summary.get("target_count")) @@ -95,6 +107,7 @@ def validate_target_rows(probes: dict[str, Any], summary: dict[str, Any], errors def validate_install_simulation_report(install: dict[str, Any], errors: list[str]) -> None: summary = install.get("summary", {}) if isinstance(install.get("summary", {}), dict) else {} + validate_no_raw_fields(install, "install_simulation", errors) add_error(errors, install.get("ok") is True, "native-permission-enforcement install simulation report ok must be true") add_error( errors, @@ -122,6 +135,7 @@ def validate_install_simulation_report(install: dict[str, Any], errors: list[str def validate_native_permission_report(probes: dict[str, Any], install: dict[str, Any], errors: list[str]) -> None: summary = probes.get("summary", {}) if isinstance(probes.get("summary", {}), dict) else {} + validate_no_raw_fields(probes, "runtime_permission_probes", errors) add_error(errors, probes.get("ok") is True, "native-permission-enforcement runtime probe report ok must be true") add_error( errors, diff --git a/scripts/world_class_provider_evidence.py b/scripts/world_class_provider_evidence.py index fff96f0..271e35e 100644 --- a/scripts/world_class_provider_evidence.py +++ b/scripts/world_class_provider_evidence.py @@ -2,6 +2,8 @@ import re from typing import Any +from output_review_privacy import forbidden_decision_field_paths + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows." @@ -32,6 +34,16 @@ def run_rows(execution: dict[str, Any]) -> list[dict[str, Any]]: return [item for item in runs if isinstance(item, dict)] if isinstance(runs, list) else [] +def validate_no_raw_fields(execution: dict[str, Any], errors: list[str]) -> None: + blocked = forbidden_decision_field_paths(execution, "output_execution_runs") + add_error( + errors, + not blocked, + "provider-holdout output execution report must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked[:8]), + ) + + def positive_duration(value: Any) -> bool: return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 @@ -78,6 +90,7 @@ def validate_provider_execution_report( ) -> None: execution_summary = summary(execution) runs = run_rows(execution) + validate_no_raw_fields(execution, errors) add_error(errors, execution.get("ok") is True, "provider-holdout output execution report ok must be true") add_error(errors, bool(runs), "provider-holdout output execution report must include run rows") diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 34b33c0..06cdfde 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -426,6 +426,28 @@ def assert_external_contract_artifact_validation() -> None: assert forged_provider["status"] == "fail", forged_provider assert any("matching provider, model, timing" in error for error in forged_provider["errors"]), forged_provider["errors"] + write_provider_artifact(skill_root, complete=True) + raw_execution = json.loads((skill_root / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) + raw_execution["runs"][0]["raw_provider_prompt"] = "verbatim provider prompt must not be accepted" + raw_execution["runs"][0]["output"] = "raw model output must not be accepted" + raw_execution["run_notes"] = [{"messages": ["raw transcript-like exchange"]}] + write_json(skill_root / "reports" / "output_execution_runs.json", raw_execution) + raw_provider = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert raw_provider["status"] == "fail", raw_provider + assert any("output_execution_runs.runs[0].raw_provider_prompt" in error for error in raw_provider["errors"]), ( + raw_provider["errors"] + ) + assert any("output_execution_runs.runs[0].output" in error for error in raw_provider["errors"]), raw_provider["errors"] + assert any("output_execution_runs.run_notes[0].messages" in error for error in raw_provider["errors"]), ( + raw_provider["errors"] + ) + permission_entry = {"key": "native-permission-enforcement", "category": "external"} write_native_permission_artifacts(skill_root, complete=True) permission_valid = validate_payload( @@ -453,6 +475,38 @@ def assert_external_contract_artifact_validation() -> None: assert any("native target rows must match summary.native_enforcement_count" in error for error in permission_forged["errors"]), ( permission_forged["errors"] ) + write_native_permission_artifacts(skill_root, complete=True) + raw_probe = json.loads((skill_root / "reports" / "runtime_permission_probes.json").read_text(encoding="utf-8")) + raw_probe["targets"][0]["messages"] = ["raw target client transcript must not be accepted"] + write_json(skill_root / "reports" / "runtime_permission_probes.json", raw_probe) + raw_permission_probe = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert raw_permission_probe["status"] == "fail", raw_permission_probe + assert any("runtime_permission_probes.targets[0].messages" in error for error in raw_permission_probe["errors"]), ( + raw_permission_probe["errors"] + ) + + write_native_permission_artifacts(skill_root, complete=True) + raw_install = json.loads((skill_root / "reports" / "install_simulation.json").read_text(encoding="utf-8")) + raw_install["checks"][0]["raw_prompt"] = "raw installer prompt must not be accepted" + write_json(skill_root / "reports" / "install_simulation.json", raw_install) + raw_permission_install = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert raw_permission_install["status"] == "fail", raw_permission_install + assert any("install_simulation.checks[0].raw_prompt" in error for error in raw_permission_install["errors"]), ( + raw_permission_install["errors"] + ) + write_native_permission_artifacts(skill_root, complete=False) permission_invalid = validate_payload( native_permission_submission(skill_root),