frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Align output review import privacy guard

Browse Source
This commit is contained in:
yaojingang
2026-06-16 23:55:40 +08:00
parent 44c6abc4f8
commit 3dfac4bead
8 changed files with 66 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/context_budget.json
+6 -6
View File
@@ -6,15 +6,15 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1070383,
"other_text_tokens": 1070810,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1071180,
"deferred_resource_tokens": 495462,
"estimated_total_text_tokens": 1071607,
"deferred_resource_tokens": 495533,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435364,
"estimated_tokens": 435435,
"file_count": 139
},
{
@@ -36,7 +36,7 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435364,
"estimated_tokens": 435435,
"file_count": 139
}
],
@@ -59,7 +59,7 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 435364,
"estimated_tokens": 435435,
"file_count": 139,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 990 | 797 | 495462 | `governed` | scripts:435364 | 131.3 | - | ok |
| root | `.` | `production` | 1000 | 990 | 797 | 495533 | `governed` | scripts:435435 | 131.3 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+3 -3
View File
@@ -8,11 +8,11 @@
"budget_limit": 1000,
"initial_tokens": 990,
"skill_body_tokens": 797,
"deferred_resource_tokens": 495462,
"deferred_resource_tokens": 495533,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 435364,
"estimated_tokens": 435435,
"file_count": 139
}
],
@@ -35,7 +35,7 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 435364,
"estimated_tokens": 435435,
"file_count": 139,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 230,
"package_sha256": "2bd5ae0684cf7508cce7611a6a840c287d35067f13e5f78f33c45667c501280c"
"package_sha256": "81880e6374f9824422252d99c7b657d49eb626c2bd8ac2f9710317fb977417a8"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `230`
- Package SHA256: `2bd5ae0684cf7508cce7611a6a840c287d35067f13e5f78f33c45667c501280c`
- Package SHA256: `81880e6374f9824422252d99c7b657d49eb626c2bd8ac2f9710317fb977417a8`
## Failures
reports/skill-os-2-review.md
+1 -1
View File
@@ -52,7 +52,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
- Review Studio now avoids over-claiming release readiness when blind A/B adjudication, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, three warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion.
- Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning.
- Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case.
- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, recursively rejects raw prompt/output/transcript/message and answer-key fields, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist.
- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, recursively rejects raw prompt/output/transcript/message, credential, secret, token, and answer-key fields case-insensitively, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist.
- Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue.
- Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes.
scripts/import_output_review_decisions.py
+13 -1
View File
@@ -24,6 +24,7 @@ SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file."
RAW_CONTENT_FIELDS = {
"api_key",
"prompt",
"prompts",
"input",
@@ -35,18 +36,28 @@ RAW_CONTENT_FIELDS = {
"message",
"messages",
"user_message",
"user_messages",
"assistant_message",
"assistant_messages",
"model_output",
"baseline_output",
"with_skill_output",
"raw_content",
"raw_prompt",
"raw_provider_prompt",
"raw_user_content",
"raw_output",
"credential",
"credentials",
"secret",
"secrets",
"token",
}
ANSWER_KEY_FIELDS = {
"expected",
"expected_winner",
"expected_winner_role",
"expected_winner_variant",
"answer_key",
"label",
@@ -148,7 +159,8 @@ def forbidden_field_paths(value: Any, prefix: str) -> list[str]:
for key, child in value.items():
key_text = str(key).strip()
child_path = f"{prefix}.{key_text}"
if key_text in RAW_CONTENT_FIELDS or key_text in ANSWER_KEY_FIELDS:
normalized_key = key_text.casefold()
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
found.append(child_path)
found.extend(forbidden_field_paths(child, child_path))
elif isinstance(value, list):
tests/verify_output_review_adjudication.py
+40 -2
View File
@@ -324,7 +324,7 @@ def main() -> None:
{
"case_id": answer_key["answers"][0]["case_id"],
"winner_variant": "A",
"metadata": {"raw_output": "nested raw output must not be imported"},
"metadata": {"Raw_Output": "nested raw output must not be imported"},
}
],
},
@@ -349,9 +349,47 @@ def main() -> None:
nested_private_payload = json.loads(nested_private_proc.stdout)
assert nested_private_proc.returncode == 2, nested_private_payload
assert nested_private_payload["ok"] is False, nested_private_payload
assert any("decision #1.metadata.raw_output" in failure for failure in nested_private_payload["failures"]), nested_private_payload
assert any("decision #1.metadata.Raw_Output" in failure for failure in nested_private_payload["failures"]), nested_private_payload
assert not (tmp_root / "nested_private_decisions.json").exists(), nested_private_payload
credential_source = tmp_root / "credential_source.json"
credential_source.write_text(
json.dumps(
{
"reviewer": "Yao QA",
"reviewed_at": "2026-06-13",
"decisions": [
{
"case_id": answer_key["answers"][0]["case_id"],
"winner_variant": "A",
"metadata": {"API_KEY": "credential material must not be imported"},
}
],
},
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
credential_proc = run(
[
str(IMPORTER),
"--input",
str(credential_source),
"--blind-pack",
str(blind_pack_json),
"--output-json",
str(tmp_root / "credential_decisions.json"),
],
check=False,
)
credential_payload = json.loads(credential_proc.stdout)
assert credential_proc.returncode == 2, credential_payload
assert credential_payload["ok"] is False, credential_payload
assert any("decision #1.metadata.API_KEY" in failure for failure in credential_payload["failures"]), credential_payload
assert not (tmp_root / "credential_decisions.json").exists(), credential_payload
unknown_case = tmp_root / "unknown_case.json"
unknown_case.write_text(
json.dumps(