Align human evidence privacy guard
This commit is contained in:
@@ -6,15 +6,15 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 797,
|
||||
"other_text_tokens": 1070804,
|
||||
"other_text_tokens": 1070969,
|
||||
"estimated_initial_load_tokens": 990,
|
||||
"estimated_total_text_tokens": 1071601,
|
||||
"deferred_resource_tokens": 495533,
|
||||
"estimated_total_text_tokens": 1071766,
|
||||
"deferred_resource_tokens": 495581,
|
||||
"deferred_resource_warn_threshold": 120000,
|
||||
"deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 435435,
|
||||
"estimated_tokens": 435483,
|
||||
"file_count": 139
|
||||
},
|
||||
{
|
||||
@@ -36,7 +36,7 @@
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 435435,
|
||||
"estimated_tokens": 435483,
|
||||
"file_count": 139
|
||||
}
|
||||
],
|
||||
@@ -59,7 +59,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 435435,
|
||||
"estimated_tokens": 435483,
|
||||
"file_count": 139,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 495533 | `governed` | scripts:435435 | 131.3 | - | ok |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 495581 | `governed` | scripts:435483 | 131.3 | - | ok |
|
||||
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
|
||||
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
"budget_limit": 1000,
|
||||
"initial_tokens": 990,
|
||||
"skill_body_tokens": 797,
|
||||
"deferred_resource_tokens": 495533,
|
||||
"deferred_resource_tokens": 495581,
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 435435,
|
||||
"estimated_tokens": 435483,
|
||||
"file_count": 139
|
||||
}
|
||||
],
|
||||
@@ -35,7 +35,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 435435,
|
||||
"estimated_tokens": 435483,
|
||||
"file_count": 139,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 230,
|
||||
"package_sha256": "81880e6374f9824422252d99c7b657d49eb626c2bd8ac2f9710317fb977417a8"
|
||||
"package_sha256": "e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `230`
|
||||
- Package SHA256: `81880e6374f9824422252d99c7b657d49eb626c2bd8ac2f9710317fb977417a8`
|
||||
- Package SHA256: `e591effaea982d2b595058bbd03ada2e8fd57a0067f65aeb3fb38b5015bb5a16`
|
||||
|
||||
## Failures
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
|
||||
- World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts.
|
||||
- World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps.
|
||||
- World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `<evidence-key>.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields.
|
||||
- World-Class Human Evidence Guard v0 so human-adjudication decision artifacts recursively reject raw content and answer-key fields before blind A/B review evidence can be accepted.
|
||||
- World-Class Human Evidence Guard v0 so human-adjudication decision artifacts recursively reject raw content, credential, secret, token, and answer-key fields before blind A/B review evidence can be accepted.
|
||||
- World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance.
|
||||
- World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free.
|
||||
- World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture.
|
||||
|
||||
@@ -8,9 +8,12 @@ SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to valid
|
||||
|
||||
REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
|
||||
RAW_CONTENT_FIELDS = {
|
||||
"api_key",
|
||||
"assistant_message",
|
||||
"assistant_messages",
|
||||
"baseline_output",
|
||||
"credential",
|
||||
"credentials",
|
||||
"input",
|
||||
"inputs",
|
||||
"message",
|
||||
@@ -23,6 +26,11 @@ RAW_CONTENT_FIELDS = {
|
||||
"raw_content",
|
||||
"raw_output",
|
||||
"raw_prompt",
|
||||
"raw_provider_prompt",
|
||||
"raw_user_content",
|
||||
"secret",
|
||||
"secrets",
|
||||
"token",
|
||||
"transcript",
|
||||
"transcripts",
|
||||
"user_message",
|
||||
@@ -79,7 +87,8 @@ def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
|
||||
for key, child in value.items():
|
||||
key_text = str(key).strip()
|
||||
child_path = f"{prefix}.{key_text}"
|
||||
if key_text.lower() in RAW_CONTENT_FIELDS or key_text.lower() in ANSWER_KEY_FIELDS:
|
||||
normalized_key = key_text.casefold()
|
||||
if normalized_key in RAW_CONTENT_FIELDS or normalized_key in ANSWER_KEY_FIELDS:
|
||||
found.append(child_path)
|
||||
found.extend(forbidden_decision_field_paths(child, child_path))
|
||||
elif isinstance(value, list):
|
||||
|
||||
@@ -249,3 +249,9 @@ def assert_human_contract_artifact_validation() -> None:
|
||||
forged_decisions["decisions"][0]["metadata"] = {"raw_output": "verbatim model answer must not ship"}
|
||||
write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions)
|
||||
assert_human_submission_error(skill_root, entry, "decisions[1].metadata.raw_output")
|
||||
|
||||
write_human_artifacts(skill_root, complete=True, reviewer="Yao QA")
|
||||
forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8"))
|
||||
forged_decisions["decisions"][0]["metadata"] = {"API_KEY": "credential material must not ship"}
|
||||
write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions)
|
||||
assert_human_submission_error(skill_root, entry, "decisions[1].metadata.API_KEY")
|
||||
|
||||
Reference in New Issue
Block a user