#!/usr/bin/env python3 import hashlib import json import re from pathlib import Path from typing import Any SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks." REQUIRED_TOP_LEVEL = [ "schema_version", "evidence_key", "template_only", "category", "source_type", "submitted_by", "submitted_at", "summary", "artifact_refs", "provenance", "privacy", "anti_overclaim", "attestation", ] REQUIRED_PRIVACY_FALSE = [ "raw_user_content_included", "raw_provider_prompt_included", "credentials_included", "secrets_included", ] REQUIRED_ANTI_OVERCLAIM_FALSE = [ "planned_work_counts_as_evidence", "metadata_fallback_counts_as_native_enforcement", "pending_review_counts_as_human_decision", "local_command_runner_counts_as_provider_model", ] REQUIRED_ATTESTATION_TRUE = [ "real_external_or_human_evidence", "reviewer_or_operator_identity_present", "artifact_refs_reviewed", "privacy_contract_satisfied", ] EXPECTED_SOURCE_TYPES = { "provider-holdout": "provider-output-eval", "human-adjudication": "blind-ab-review", "native-permission-enforcement": "runtime-permission-guard", "native-client-telemetry": "native-client-telemetry", } SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") SUBMITTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$") DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"} REQUIRED_REAL_ARTIFACT_PATHS = { "provider-holdout": {"reports/output_execution_runs.json"}, "human-adjudication": { "reports/output_review_adjudication.json", "reports/output_review_decisions.json", }, "native-permission-enforcement": { "reports/runtime_permission_probes.json", "reports/install_simulation.json", }, "native-client-telemetry": { "reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json", }, } PLACEHOLDER_FRAGMENTS = ( "YYYY-MM-DD", "name or team handle", "operator with provider credentials", "human reviewer", "target client or installer integrator", "Browser/Chrome/IDE/provider client integrator", "Browser/Chrome/IDE/provider client", "openai|claude|generic|vscode|other", "client or installer component", "/local/path/not/committed", ) def load_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: return {} return payload if isinstance(payload, dict) else {} def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]: if not path.exists(): return {}, "missing" try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: return {}, "invalid-json" if not isinstance(payload, dict): return {}, "invalid-json" return payload, "present" def rel_path(path: Path, root: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) except ValueError: return str(path.resolve()) def add_error(errors: list[str], condition: bool, message: str) -> None: if not condition: errors.append(message) def has_placeholder_text(value: Any) -> bool: text = str(value or "").strip() if not text: return False lowered = text.lower() return any(fragment.lower() in lowered for fragment in PLACEHOLDER_FRAGMENTS) def require_real_text(errors: list[str], value: Any, field: str) -> None: text = str(value or "").strip() add_error(errors, bool(text), f"{field} is required") add_error(errors, not has_placeholder_text(text), f"{field} must not use template placeholder text") def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]: text = raw_path.strip() if any(token in text for token in ("<", ">", "*", "?")): return None, "artifact_refs path must be concrete, not a placeholder or glob" candidate = Path(text) if candidate.is_absolute(): return None, "artifact_refs path must be relative to the skill directory" if ".." in candidate.parts: return None, "artifact_refs path must not escape the skill directory" resolved = (root / candidate).resolve() try: resolved.relative_to(root.resolve()) except ValueError: return None, "artifact_refs path must stay inside the skill directory" return resolved, None def validate_artifact_refs( payload: dict[str, Any], errors: list[str], *, root: Path, template_expected: bool, ) -> dict[str, int]: refs = payload.get("artifact_refs") add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference") required_paths = REQUIRED_REAL_ARTIFACT_PATHS.get(str(payload.get("evidence_key", "")), set()) observed_paths: set[str] = set() stats = { "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, "artifact_existing_count": 0, "artifact_sha256_verified_count": 0, "required_artifact_count": len(required_paths) if not template_expected else 0, "required_artifact_verified_count": 0, } if not isinstance(refs, list): return stats for index, ref in enumerate(refs): if not isinstance(ref, dict): errors.append(f"artifact_refs[{index}] must be an object") continue path_text = str(ref.get("path", "")).strip() add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required") add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required") add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content") if template_expected or not path_text: continue candidate = Path(path_text) if not candidate.is_absolute() and ".." not in candidate.parts and not any( token in path_text for token in ("<", ">", "*", "?") ): observed_paths.add(candidate.as_posix()) resolved, path_error = resolve_artifact_path(path_text, root) if path_error: errors.append(f"artifact_refs[{index}].path {path_error}") continue rel = rel_path(resolved, root) if rel in DISALLOWED_REAL_ARTIFACTS: errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs") if not resolved.exists() or not resolved.is_file(): errors.append(f"artifact_refs[{index}].path does not exist as a local file") continue stats["artifact_existing_count"] += 1 declared = str(ref.get("sha256", "")).strip() if not declared: errors.append(f"artifact_refs[{index}].sha256 is required for a real submission") continue if not SHA256_RE.match(declared): errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest") continue actual = sha256_file(resolved) if actual.lower() != declared.lower(): errors.append(f"artifact_refs[{index}].sha256 does not match local artifact") continue stats["artifact_sha256_verified_count"] += 1 if rel in required_paths: stats["required_artifact_verified_count"] += 1 if not template_expected and required_paths: missing_required = sorted(required_paths - observed_paths) for path in missing_required: errors.append(f"artifact_refs must include required evidence artifact {path}") if not missing_required and stats["required_artifact_verified_count"] < len(required_paths): errors.append("all required evidence artifacts must have verified sha256 digests") return stats def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None: key = str(payload.get("evidence_key", "")) template_expected = payload.get("template_only") is True provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} if key == "provider-holdout": add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required") add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required") add_error( errors, provenance.get("credential_material_committed") is False, "provider-holdout must attest credential_material_committed is false", ) elif key == "human-adjudication": add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required") if not template_expected: require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer") add_error( errors, provenance.get("answer_key_opened_after_decisions") is True, "human-adjudication must attest answer_key_opened_after_decisions is true", ) elif key == "native-permission-enforcement": add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required") if not template_expected: require_real_text(errors, provenance.get("target", ""), "native-permission-enforcement provenance.target") add_error( errors, bool(str(provenance.get("guard_location", "")).strip()), "native-permission-enforcement provenance.guard_location is required", ) if not template_expected: require_real_text( errors, provenance.get("guard_location", ""), "native-permission-enforcement provenance.guard_location", ) add_error( errors, str(provenance.get("guard_scope", "")).strip() in {"target-client-native", "external-installer-runtime-guard"}, "native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard", ) add_error( errors, provenance.get("guard_blocks_undeclared_capability") is True, "native-permission-enforcement must attest guard_blocks_undeclared_capability is true", ) add_error( errors, provenance.get("metadata_fallback_retained_for_other_targets") is True, "native-permission-enforcement must retain metadata fallback for non-native targets", ) elif key == "native-client-telemetry": add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required") if not template_expected: require_real_text(errors, provenance.get("client", ""), "native-client-telemetry provenance.client") if str(provenance.get("native_host_manifest", "")).strip(): require_real_text( errors, provenance.get("native_host_manifest", ""), "native-client-telemetry provenance.native_host_manifest", ) add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external") add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only") def validate_payload( payload: dict[str, Any], entry: dict[str, Any], *, path: Path, root: Path, template_expected: bool, ) -> dict[str, Any]: errors: list[str] = [] for key in REQUIRED_TOP_LEVEL: add_error(errors, key in payload, f"missing {key}") evidence_key = str(entry.get("key", "")) add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0") add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}") add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}") add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}") add_error( errors, payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key), f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}", ) add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required") add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required") add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required") if not template_expected: require_real_text(errors, payload.get("submitted_by", ""), "submitted_by") add_error( errors, bool(SUBMITTED_AT_RE.match(str(payload.get("submitted_at", "")).strip())), "submitted_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", ) artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected) privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {} for key in REQUIRED_PRIVACY_FALSE: add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false") anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {} for key in REQUIRED_ANTI_OVERCLAIM_FALSE: add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false") validate_evidence_specific(payload, errors) attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} if not template_expected: for key in REQUIRED_ATTESTATION_TRUE: add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission") return { "path": rel_path(path, root), "evidence_key": evidence_key, "status": "pass" if not errors else "fail", "template_only": template_expected, "artifact_integrity": artifact_integrity, "errors": errors, }