frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Share output review privacy guard

Browse Source
This commit is contained in:
yaojingang
2026-06-17 00:07:59 +08:00
parent 190700c7d5
commit 1945775f93
14 changed files with 145 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/architecture_maintainability.json
+4 -4
View File
@@ -1,13 +1,13 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-17",
"skill_dir": ".",
"summary": {
"python_file_count": 214,
"script_file_count": 139,
"python_file_count": 215,
"script_file_count": 140,
"test_file_count": 75,
"internal_module_count": 55,
"internal_module_count": 56,
"cli_script_count": 86,
"command_handler_count": 68,
"entrypoint_command_handler_count": 18,
reports/architecture_maintainability.md
+4 -4
View File
@@ -1,14 +1,14 @@
# Architecture Maintainability
Generated at: `2026-06-16`
Generated at: `2026-06-17`
## Summary
- decision: `pass`
- python files: `214`
- scripts: `139`
- python files: `215`
- scripts: `140`
- tests: `75`
- internal modules: `55`
- internal modules: `56`
- CLI scripts: `86`
- Yao CLI command handlers: `68`
- entrypoint command handlers: `18`
reports/context_budget.json
+10 -10
View File
@@ -6,16 +6,16 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1070976,
"other_text_tokens": 1071125,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1071773,
"deferred_resource_tokens": 495581,
"estimated_total_text_tokens": 1071922,
"deferred_resource_tokens": 495322,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435483,
"file_count": 139
"estimated_tokens": 435224,
"file_count": 140
},
{
"path": "references",
@@ -36,8 +36,8 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435483,
"file_count": 139
"estimated_tokens": 435224,
"file_count": 140
}
],
"deferred_resource_governance": {
@@ -59,14 +59,14 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 435483,
"file_count": 139,
"estimated_tokens": 435224,
"file_count": 140,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 675,
"relevant_file_count": 677,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 131.3
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 990 | 797 | 495581 | `governed` | scripts:435483 | 131.3 | - | ok |
| root | `.` | `production` | 1000 | 990 | 797 | 495322 | `governed` | scripts:435224 | 131.3 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+6 -6
View File
@@ -1,5 +1,5 @@
{
"generated_at": "2026-06-16",
"generated_at": "2026-03-31",
"targets": [
{
"label": "root",
@@ -8,12 +8,12 @@
"budget_limit": 1000,
"initial_tokens": 990,
"skill_body_tokens": 797,
"deferred_resource_tokens": 495581,
"deferred_resource_tokens": 495322,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435483,
"file_count": 139
"estimated_tokens": 435224,
"file_count": 140
}
],
"deferred_resource_governance": {
@@ -35,8 +35,8 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 435483,
"file_count": 139,
"estimated_tokens": 435224,
"file_count": 140,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
reports/python_compatibility.json
+8 -2
View File
@@ -1,11 +1,11 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-17",
"root": ".",
"summary": {
"target_python": "3.11",
"file_count": 217,
"file_count": 218,
"issue_count": 0,
"syntax_error_count": 0,
"fstring_311_violation_count": 0,
@@ -244,6 +244,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "scripts/output_review_privacy.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "scripts/prepare_output_review_kit.py",
"ok": true,
reports/python_compatibility.md
+2 -2
View File
@@ -1,12 +1,12 @@
# Python Compatibility
Generated at: `2026-06-16`
Generated at: `2026-06-17`
## Summary
- decision: `pass`
- target python: `3.11`
- files scanned: `217`
- files scanned: `218`
- issues: `0`
- syntax errors: `0`
- f-string 3.11 violations: `0`
reports/security_trust_report.json
+24 -6
View File
@@ -2,9 +2,9 @@
"ok": true,
"skill_dir": ".",
"summary": {
"scanned_files": 230,
"script_count": 139,
"internal_module_count": 53,
"scanned_files": 231,
"script_count": 140,
"internal_module_count": 54,
"secret_findings": 0,
"dependency_files": [
"requirements-ci.txt"
@@ -22,8 +22,8 @@
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 230,
"package_sha256": "e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16"
"package_hash_file_count": 231,
"package_sha256": "fcfe5f3c722285daf14b6f83118ad71bdca21694d86d870d234d9f5df2f4e627"
},
"failures": [],
"warnings": [],
@@ -514,6 +514,20 @@
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/output_review_privacy.py",
"interface": "internal-module",
"interface_declared": true,
"interface_reason": "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively.",
"has_argparse": false,
"has_main_guard": false,
"uses_input": false,
"uses_network": false,
"uses_file_write": false,
"uses_subprocess": false,
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/prepare_output_review_kit.py",
"interface": "cli",
@@ -2027,7 +2041,7 @@
"checked_count": 86,
"passed_count": 86,
"failed_count": 0,
"skipped_count": 53,
"skipped_count": 54,
"failed_scripts": [],
"results": [
{
@@ -2936,6 +2950,10 @@
"path": "scripts/html_rendering.py",
"reason": "internal module"
},
{
"path": "scripts/output_review_privacy.py",
"reason": "internal module"
},
{
"path": "scripts/review_studio_action_evidence.py",
"reason": "internal module"
reports/security_trust_report.md
+6 -5
View File
@@ -1,9 +1,9 @@
# Security Trust Report
- OK: `True`
- Scanned files: `230`
- Scripts: `139`
- Internal script modules: `53`
- Scanned files: `231`
- Scripts: `140`
- Internal script modules: `54`
- Secret findings: `0`
- Network-capable scripts: `3`
- Network policy covered scripts: `3`
@@ -15,8 +15,8 @@
- CLI help smoke failures: `0`
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `230`
- Package SHA256: `e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16`
- Package hash files: `231`
- Package SHA256: `fcfe5f3c722285daf14b6f83118ad71bdca21694d86d870d234d9f5df2f4e627`
## Failures
@@ -96,6 +96,7 @@
| scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/output_review_privacy.py | internal-module | True | False | False | False | False | False | False | Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively. |
| scripts/prepare_output_review_kit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/prepare_world_class_submission_kit.py | cli | True | True | True | False | False | True | False | Prepares editable world-class evidence intake packets without counting drafts as accepted evidence. |
| scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
reports/skill-os-2-review.md
+4 -3
View File
@@ -22,6 +22,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
- World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps.
- World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `<evidence-key>.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields.
- World-Class Human Evidence Guard v0 so human-adjudication decision artifacts recursively reject raw content, credential, secret, token, and answer-key fields before blind A/B review evidence can be accepted.
- Output Review Privacy Guard v0 so the blind-review importer and world-class human evidence validator share one recursive blocked-field contract for raw content, credential, secret, token, and answer-key leakage.
- World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance.
- World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free.
- World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture.
@@ -64,13 +65,13 @@ This is still not the final world-class state. Target-native behavior contracts
| --- | --- | --- |
| Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed |
| Target Compiler | `scripts/compile_skill.py`, `scripts/compile_skill_targets.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed |
| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed |
| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/output_review_privacy.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed |
| Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed |
| Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed |
| Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed |
| World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed |
| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_human_evidence.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, recursive human decision privacy and answer-key validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_human_evidence.py`, `scripts/output_review_privacy.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, recursive human decision privacy and answer-key validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed |
| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed |
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
@@ -131,7 +132,7 @@ Next move: add real client or installer permission enforcement integration.
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden |
| Runtime Conformance | `5 / 5` targets passing |
| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts |
| Trust | `0` secret findings, `1` pinned dependency file, `53` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `139` scripts, source-contract hash scope explicit |
| Trust | `0` secret findings, `1` pinned dependency file, `54` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `140` scripts, source-contract hash scope explicit |
| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired |
| Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility |
| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility |
scripts/import_output_review_decisions.py
+2 -60
View File
@@ -17,55 +17,13 @@ from adjudicate_output_review import (
normalize_variant,
pair_index,
)
from output_review_privacy import forbidden_decision_field_paths
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file."
RAW_CONTENT_FIELDS = {
"api_key",
"prompt",
"prompts",
"input",
"inputs",
"output",
"outputs",
"transcript",
"transcripts",
"message",
"messages",
"user_message",
"user_messages",
"assistant_message",
"assistant_messages",
"model_output",
"baseline_output",
"with_skill_output",
"raw_content",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"raw_output",
"credential",
"credentials",
"secret",
"secrets",
"token",
}
ANSWER_KEY_FIELDS = {
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"answer_key",
"label",
"variant_label",
"baseline_label",
"with_skill_label",
}
def display_path(path: Path) -> str:
try:
@@ -153,22 +111,6 @@ def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None
return round(parsed, 3), None
def forbidden_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
normalized_key = key_text.casefold()
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
found.append(child_path)
found.extend(forbidden_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_field_paths(child, f"{prefix}[{index}]"))
return found
def normalize_decisions(
source_items: list[dict[str, Any]],
case_ids: set[str],
@@ -180,7 +122,7 @@ def normalize_decisions(
if not isinstance(item, dict):
failures.append(f"decision #{index} must be an object")
continue
blocked_fields = forbidden_field_paths(item, f"decision #{index}")
blocked_fields = forbidden_decision_field_paths(item, f"decision #{index}")
if blocked_fields:
failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}")
case_id = str(item.get("case_id", "")).strip()
scripts/output_review_privacy.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
from typing import Any
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively."
RAW_CONTENT_FIELDS = {
"api_key",
"assistant_message",
"assistant_messages",
"baseline_output",
"credential",
"credentials",
"input",
"inputs",
"message",
"messages",
"model_output",
"output",
"outputs",
"prompt",
"prompts",
"raw_content",
"raw_output",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"secret",
"secrets",
"token",
"transcript",
"transcripts",
"user_message",
"user_messages",
"with_skill_output",
}
ANSWER_KEY_FIELDS = {
"answer_key",
"baseline_label",
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"label",
"variant_label",
"with_skill_label",
}
BLOCKED_DECISION_FIELDS = RAW_CONTENT_FIELDS | ANSWER_KEY_FIELDS
def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
if key_text.casefold() in BLOCKED_DECISION_FIELDS:
found.append(child_path)
found.extend(forbidden_decision_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]"))
return found
scripts/world_class_human_evidence.py
+2 -57
View File
@@ -2,52 +2,13 @@
import re
from typing import Any
from output_review_privacy import forbidden_decision_field_paths
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows."
REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
RAW_CONTENT_FIELDS = {
"api_key",
"assistant_message",
"assistant_messages",
"baseline_output",
"credential",
"credentials",
"input",
"inputs",
"message",
"messages",
"model_output",
"output",
"outputs",
"prompt",
"prompts",
"raw_content",
"raw_output",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"secret",
"secrets",
"token",
"transcript",
"transcripts",
"user_message",
"user_messages",
"with_skill_output",
}
ANSWER_KEY_FIELDS = {
"answer_key",
"baseline_label",
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"label",
"variant_label",
"with_skill_label",
}
def add_error(errors: list[str], condition: bool, message: str) -> None:
@@ -81,22 +42,6 @@ def duplicate_case_ids(ids: list[str]) -> list[str]:
return duplicates
def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
found: list[str] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
normalized_key = key_text.casefold()
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
found.append(child_path)
found.extend(forbidden_decision_field_paths(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]"))
return found
def confidence_valid(value: Any) -> bool:
if value is None or value == "":
return True
tests/verify_output_review_adjudication.py
+6
View File
@@ -7,6 +7,10 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from output_review_privacy import BLOCKED_DECISION_FIELDS # noqa: E402
OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py"
ADJUDICATOR = ROOT / "scripts" / "adjudicate_output_review.py"
IMPORTER = ROOT / "scripts" / "import_output_review_decisions.py"
@@ -24,6 +28,8 @@ def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]
def main() -> None:
assert {"api_key", "raw_provider_prompt", "expected_winner_role"} <= BLOCKED_DECISION_FIELDS
tmp_root = ROOT / "tests" / "tmp_output_review_adjudication"
if tmp_root.exists():
shutil.rmtree(tmp_root)