frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Share output review privacy guard

Browse Source
This commit is contained in:
yaojingang
2026-06-17 00:07:59 +08:00
parent 190700c7d5
commit 1945775f93
14 changed files with 145 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/import_output_review_decisions.py
+2 -60
View File
@@ -17,55 +17,13 @@ from adjudicate_output_review import (
normalize_variant,
pair_index,
)
from output_review_privacy import forbidden_decision_field_paths
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file."
RAW_CONTENT_FIELDS = {
"api_key",
"prompt",
"prompts",
"input",
"inputs",
"output",
"outputs",
"transcript",
"transcripts",
"message",
"messages",
"user_message",
"user_messages",
"assistant_message",
"assistant_messages",
"model_output",
"baseline_output",
"with_skill_output",
"raw_content",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"raw_output",
"credential",
"credentials",
"secret",
"secrets",
"token",
}
ANSWER_KEY_FIELDS = {
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"answer_key",
"label",
"variant_label",
"baseline_label",
"with_skill_label",
}
def display_path(path: Path) -> str:
try:
@@ -153,22 +111,6 @@ def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None
return round(parsed, 3), None
def forbidden_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
normalized_key = key_text.casefold()
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
found.append(child_path)
found.extend(forbidden_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_field_paths(child, f"{prefix}[{index}]"))
return found
def normalize_decisions(
source_items: list[dict[str, Any]],
case_ids: set[str],
@@ -180,7 +122,7 @@ def normalize_decisions(
if not isinstance(item, dict):
failures.append(f"decision #{index} must be an object")
continue
blocked_fields = forbidden_field_paths(item, f"decision #{index}")
blocked_fields = forbidden_decision_field_paths(item, f"decision #{index}")
if blocked_fields:
failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}")
case_id = str(item.get("case_id", "")).strip()
scripts/output_review_privacy.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
from typing import Any
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively."
RAW_CONTENT_FIELDS = {
"api_key",
"assistant_message",
"assistant_messages",
"baseline_output",
"credential",
"credentials",
"input",
"inputs",
"message",
"messages",
"model_output",
"output",
"outputs",
"prompt",
"prompts",
"raw_content",
"raw_output",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"secret",
"secrets",
"token",
"transcript",
"transcripts",
"user_message",
"user_messages",
"with_skill_output",
}
ANSWER_KEY_FIELDS = {
"answer_key",
"baseline_label",
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"label",
"variant_label",
"with_skill_label",
}
BLOCKED_DECISION_FIELDS = RAW_CONTENT_FIELDS | ANSWER_KEY_FIELDS
def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
if key_text.casefold() in BLOCKED_DECISION_FIELDS:
found.append(child_path)
found.extend(forbidden_decision_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]"))
return found
scripts/world_class_human_evidence.py
+2 -57
View File
@@ -2,52 +2,13 @@
import re
from typing import Any
from output_review_privacy import forbidden_decision_field_paths
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows."
REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
RAW_CONTENT_FIELDS = {
"api_key",
"assistant_message",
"assistant_messages",
"baseline_output",
"credential",
"credentials",
"input",
"inputs",
"message",
"messages",
"model_output",
"output",
"outputs",
"prompt",
"prompts",
"raw_content",
"raw_output",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"secret",
"secrets",
"token",
"transcript",
"transcripts",
"user_message",
"user_messages",
"with_skill_output",
}
ANSWER_KEY_FIELDS = {
"answer_key",
"baseline_label",
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"label",
"variant_label",
"with_skill_label",
}
def add_error(errors: list[str], condition: bool, message: str) -> None:
@@ -81,22 +42,6 @@ def duplicate_case_ids(ids: list[str]) -> list[str]:
return duplicates
def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
normalized_key = key_text.casefold()
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
found.append(child_path)
found.extend(forbidden_decision_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]"))
return found
def confidence_valid(value: Any) -> bool:
if value is None or value == "":
return True