yao-meta-skill/scripts/world_class_evidence_contract.py

#!/usr/bin/env python3
import hashlib
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from output_review_privacy import BLOCKED_DECISION_FIELDS
from world_class_human_evidence import validate_human_adjudication_report
from world_class_native_permission_evidence import validate_native_permission_report
from world_class_native_telemetry_evidence import validate_native_telemetry_report
from world_class_provider_evidence import validate_provider_execution_report


SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks."

REQUIRED_TOP_LEVEL = [
    "schema_version",
    "evidence_key",
    "template_only",
    "category",
    "source_type",
    "submitted_by",
    "submitted_at",
    "summary",
    "artifact_refs",
    "provenance",
    "privacy",
    "anti_overclaim",
    "attestation",
]
REQUIRED_PRIVACY_FALSE = [
    "raw_user_content_included",
    "raw_provider_prompt_included",
    "credentials_included",
    "secrets_included",
]
REQUIRED_ANTI_OVERCLAIM_FALSE = [
    "planned_work_counts_as_evidence",
    "metadata_fallback_counts_as_native_enforcement",
    "pending_review_counts_as_human_decision",
    "local_command_runner_counts_as_provider_model",
]
REQUIRED_ATTESTATION_TRUE = [
    "real_external_or_human_evidence",
    "reviewer_or_operator_identity_present",
    "artifact_refs_reviewed",
    "privacy_contract_satisfied",
    "ledger_reviewer_approved",
]
EXPECTED_SOURCE_TYPES = {
    "provider-holdout": "provider-output-eval",
    "human-adjudication": "blind-ab-review",
    "native-permission-enforcement": "runtime-permission-guard",
    "native-client-telemetry": "native-client-telemetry",
}
SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
SUBMITTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"}
REQUIRED_REAL_ARTIFACT_PATHS = {
    "provider-holdout": {"reports/output_execution_runs.json"},
    "human-adjudication": {
        "reports/output_review_adjudication.json",
        "reports/output_review_decisions.json",
    },
    "native-permission-enforcement": {
        "reports/runtime_permission_probes.json",
        "reports/install_simulation.json",
    },
    "native-client-telemetry": {
        "reports/adoption_drift_report.json",
        "reports/telemetry_hook_recipes.json",
    },
}
EXPECTED_REAL_ARTIFACT_KINDS = {
    "provider-holdout": {
        "reports/output_execution_runs.json": "aggregate-report",
    },
    "human-adjudication": {
        "reports/output_review_adjudication.json": "adjudication-report",
        "reports/output_review_decisions.json": "review-decisions",
    },
    "native-permission-enforcement": {
        "reports/runtime_permission_probes.json": "runtime-probe-report",
        "reports/install_simulation.json": "installer-enforcement-report",
    },
    "native-client-telemetry": {
        "reports/adoption_drift_report.json": "adoption-drift-report",
        "reports/telemetry_hook_recipes.json": "hook-recipes",
    },
}
PLACEHOLDER_FRAGMENTS = (
    "YYYY-MM-DD",
    "name or team handle",
    "operator with provider credentials",
    "human reviewer",
    "target client or installer integrator",
    "Browser/Chrome/IDE/provider client integrator",
    "Browser/Chrome/IDE/provider client",
    "openai|claude|generic|vscode|other",
    "client or installer component",
    "/local/path/not/committed",
)
FORBIDDEN_REAL_SUBMISSION_FIELDS = BLOCKED_DECISION_FIELDS


def load_json(path: Path) -> dict[str, Any]:
    if not path.exists():
        return {}
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return {}
    return payload if isinstance(payload, dict) else {}


def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]:
    if not path.exists():
        return {}, "missing"
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return {}, "invalid-json"
    if not isinstance(payload, dict):
        return {}, "invalid-json"
    return payload, "present"


def rel_path(path: Path, root: Path) -> str:
    try:
        return str(path.resolve().relative_to(root.resolve()))
    except ValueError:
        return str(path.resolve())


def add_error(errors: list[str], condition: bool, message: str) -> None:
    if not condition:
        errors.append(message)


def has_placeholder_text(value: Any) -> bool:
    text = str(value or "").strip()
    if not text:
        return False
    lowered = text.lower()
    return any(fragment.lower() in lowered for fragment in PLACEHOLDER_FRAGMENTS)


def require_real_text(errors: list[str], value: Any, field: str) -> None:
    text = str(value or "").strip()
    add_error(errors, bool(text), f"{field} is required")
    add_error(errors, not has_placeholder_text(text), f"{field} must not use template placeholder text")


def parse_audit_timestamp(value: Any) -> datetime | None:
    text = str(value or "").strip()
    if not SUBMITTED_AT_RE.match(text):
        return None
    if "T" in text:
        return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc)
    return datetime.fromisoformat(text).replace(tzinfo=timezone.utc)


def forbidden_submission_field_paths(value: Any, prefix: str = "$") -> list[str]:
    found: list[str] = []
    if isinstance(value, dict):
        for key, child in value.items():
            key_text = str(key)
            child_path = f"{prefix}.{key_text}"
            if key_text.strip().lower() in FORBIDDEN_REAL_SUBMISSION_FIELDS:
                found.append(child_path)
            found.extend(forbidden_submission_field_paths(child, child_path))
    elif isinstance(value, list):
        for index, child in enumerate(value):
            found.extend(forbidden_submission_field_paths(child, f"{prefix}[{index}]"))
    return found


def validate_real_submission_file_identity(
    payload: dict[str, Any],
    errors: list[str],
    *,
    path: Path,
    template_expected: bool,
) -> None:
    if template_expected:
        return
    evidence_key = str(payload.get("evidence_key", "")).strip()
    expected_name = f"{evidence_key}.json"
    add_error(
        errors,
        path.name == expected_name,
        f"real submission filename must be {expected_name}",
    )


def validate_real_submission_privacy_fields(
    payload: dict[str, Any],
    errors: list[str],
    *,
    template_expected: bool,
) -> None:
    if template_expected:
        return
    blocked_paths = forbidden_submission_field_paths(payload)
    add_error(
        errors,
        not blocked_paths,
        "real submission must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: "
        + ", ".join(blocked_paths[:8]),
    )


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]:
    text = raw_path.strip()
    if any(token in text for token in ("<", ">", "*", "?")):
        return None, "artifact_refs path must be concrete, not a placeholder or glob"
    candidate = Path(text)
    if candidate.is_absolute():
        return None, "artifact_refs path must be relative to the skill directory"
    if ".." in candidate.parts:
        return None, "artifact_refs path must not escape the skill directory"
    resolved = (root / candidate).resolve()
    try:
        resolved.relative_to(root.resolve())
    except ValueError:
        return None, "artifact_refs path must stay inside the skill directory"
    return resolved, None


def validate_artifact_refs(
    payload: dict[str, Any],
    errors: list[str],
    *,
    root: Path,
    template_expected: bool,
) -> dict[str, int]:
    refs = payload.get("artifact_refs")
    add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference")
    evidence_key = str(payload.get("evidence_key", ""))
    required_paths = REQUIRED_REAL_ARTIFACT_PATHS.get(evidence_key, set())
    expected_kinds = EXPECTED_REAL_ARTIFACT_KINDS.get(evidence_key, {})
    observed_paths: set[str] = set()
    seen_artifact_paths: set[str] = set()
    stats = {
        "artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
        "artifact_existing_count": 0,
        "artifact_sha256_verified_count": 0,
        "required_artifact_count": len(required_paths) if not template_expected else 0,
        "required_artifact_verified_count": 0,
    }
    if not isinstance(refs, list):
        return stats
    for index, ref in enumerate(refs):
        if not isinstance(ref, dict):
            errors.append(f"artifact_refs[{index}] must be an object")
            continue
        path_text = str(ref.get("path", "")).strip()
        add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required")
        add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required")
        add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content")
        if template_expected or not path_text:
            continue
        candidate = Path(path_text)
        if not candidate.is_absolute() and ".." not in candidate.parts and not any(
            token in path_text for token in ("<", ">", "*", "?")
        ):
            observed_paths.add(candidate.as_posix())
        resolved, path_error = resolve_artifact_path(path_text, root)
        if path_error:
            errors.append(f"artifact_refs[{index}].path {path_error}")
            continue
        rel = rel_path(resolved, root)
        add_error(
            errors,
            rel not in seen_artifact_paths,
            f"artifact_refs[{index}].path must not duplicate another artifact reference",
        )
        seen_artifact_paths.add(rel)
        if rel in DISALLOWED_REAL_ARTIFACTS:
            errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs")
        expected_kind = expected_kinds.get(rel)
        if expected_kind:
            observed_kind = str(ref.get("kind", "")).strip()
            add_error(
                errors,
                observed_kind == expected_kind,
                f"artifact_refs[{index}].kind must be {expected_kind} for {rel}",
            )
        if not resolved.exists() or not resolved.is_file():
            errors.append(f"artifact_refs[{index}].path does not exist as a local file")
            continue
        stats["artifact_existing_count"] += 1
        declared = str(ref.get("sha256", "")).strip()
        if not declared:
            errors.append(f"artifact_refs[{index}].sha256 is required for a real submission")
            continue
        if not SHA256_RE.match(declared):
            errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest")
            continue
        actual = sha256_file(resolved)
        if actual.lower() != declared.lower():
            errors.append(f"artifact_refs[{index}].sha256 does not match local artifact")
            continue
        stats["artifact_sha256_verified_count"] += 1
        if rel in required_paths:
            stats["required_artifact_verified_count"] += 1
    if not template_expected and required_paths:
        missing_required = sorted(required_paths - observed_paths)
        for path in missing_required:
            errors.append(f"artifact_refs must include required evidence artifact {path}")
        if not missing_required and stats["required_artifact_verified_count"] < len(required_paths):
            errors.append("all required evidence artifacts must have verified sha256 digests")
    return stats


def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path]:
    refs = payload.get("artifact_refs", [])
    if not isinstance(refs, list):
        return {}
    paths: dict[str, Path] = {}
    for ref in refs:
        if not isinstance(ref, dict):
            continue
        path_text = str(ref.get("path", "")).strip()
        if not path_text:
            continue
        resolved, path_error = resolve_artifact_path(path_text, root)
        if path_error or resolved is None:
            continue
        paths[rel_path(resolved, root)] = resolved
    return paths


def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
    paths = artifact_ref_path_map(payload, root)
    execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__"))
    if not execution:
        return
    provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
    validate_provider_execution_report(execution, provenance, errors)


def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
    paths = artifact_ref_path_map(payload, root)
    adjudication = load_json(paths.get("reports/output_review_adjudication.json", root / "__missing__"))
    decisions = load_json(paths.get("reports/output_review_decisions.json", root / "__missing__"))
    if not adjudication or not decisions:
        return
    provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
    validate_human_adjudication_report(adjudication, decisions, provenance, errors)


def validate_native_permission_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
    paths = artifact_ref_path_map(payload, root)
    probes = load_json(paths.get("reports/runtime_permission_probes.json", root / "__missing__"))
    install = load_json(paths.get("reports/install_simulation.json", root / "__missing__"))
    if probes:
        validate_native_permission_report(probes, install, errors)


def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
    paths = artifact_ref_path_map(payload, root)
    adoption = load_json(paths.get("reports/adoption_drift_report.json", root / "__missing__"))
    recipes = load_json(paths.get("reports/telemetry_hook_recipes.json", root / "__missing__"))
    validate_native_telemetry_report(adoption, recipes, errors)


def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None:
    if template_expected:
        return
    evidence_key = str(payload.get("evidence_key", ""))
    if evidence_key == "provider-holdout":
        validate_provider_holdout_artifacts(payload, errors, root)
    elif evidence_key == "human-adjudication":
        validate_human_adjudication_artifacts(payload, errors, root)
    elif evidence_key == "native-permission-enforcement":
        validate_native_permission_artifacts(payload, errors, root)
    elif evidence_key == "native-client-telemetry":
        validate_native_client_telemetry_artifacts(payload, errors, root)


def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None:
    key = str(payload.get("evidence_key", ""))
    template_expected = payload.get("template_only") is True
    provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
    if key == "provider-holdout":
        add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required")
        add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required")
        add_error(
            errors,
            provenance.get("credential_material_committed") is False,
            "provider-holdout must attest credential_material_committed is false",
        )
    elif key == "human-adjudication":
        add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
        if not template_expected:
            require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer")
        add_error(
            errors,
            provenance.get("answer_key_opened_after_decisions") is True,
            "human-adjudication must attest answer_key_opened_after_decisions is true",
        )
    elif key == "native-permission-enforcement":
        add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
        if not template_expected:
            require_real_text(errors, provenance.get("target", ""), "native-permission-enforcement provenance.target")
        add_error(
            errors,
            bool(str(provenance.get("guard_location", "")).strip()),
            "native-permission-enforcement provenance.guard_location is required",
        )
        if not template_expected:
            require_real_text(
                errors,
                provenance.get("guard_location", ""),
                "native-permission-enforcement provenance.guard_location",
            )
        add_error(
            errors,
            str(provenance.get("guard_scope", "")).strip()
            in {"target-client-native", "external-installer-runtime-guard"},
            "native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard",
        )
        add_error(
            errors,
            provenance.get("guard_blocks_undeclared_capability") is True,
            "native-permission-enforcement must attest guard_blocks_undeclared_capability is true",
        )
        add_error(
            errors,
            provenance.get("metadata_fallback_retained_for_other_targets") is True,
            "native-permission-enforcement must retain metadata fallback for non-native targets",
        )
    elif key == "native-client-telemetry":
        add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required")
        if not template_expected:
            require_real_text(errors, provenance.get("client", ""), "native-client-telemetry provenance.client")
            if str(provenance.get("native_host_manifest", "")).strip():
                require_real_text(
                    errors,
                    provenance.get("native_host_manifest", ""),
                    "native-client-telemetry provenance.native_host_manifest",
                )
        add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external")
        add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only")


def validate_payload(
    payload: dict[str, Any],
    entry: dict[str, Any],
    *,
    path: Path,
    root: Path,
    template_expected: bool,
) -> dict[str, Any]:
    errors: list[str] = []
    for key in REQUIRED_TOP_LEVEL:
        add_error(errors, key in payload, f"missing {key}")
    evidence_key = str(entry.get("key", ""))
    add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0")
    add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}")
    add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}")
    add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}")
    add_error(
        errors,
        payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key),
        f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}",
    )
    add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required")
    add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required")
    add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required")
    if not template_expected:
        validate_real_submission_file_identity(payload, errors, path=path, template_expected=template_expected)
        validate_real_submission_privacy_fields(payload, errors, template_expected=template_expected)
        require_real_text(errors, payload.get("submitted_by", ""), "submitted_by")
        add_error(
            errors,
            bool(SUBMITTED_AT_RE.match(str(payload.get("submitted_at", "")).strip())),
            "submitted_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
        )
    artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected)
    privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {}
    for key in REQUIRED_PRIVACY_FALSE:
        add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false")
    anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {}
    for key in REQUIRED_ANTI_OVERCLAIM_FALSE:
        add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false")
    validate_evidence_specific(payload, errors)
    attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
    if not template_expected:
        for key in REQUIRED_ATTESTATION_TRUE:
            add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission")
        ledger_reviewer = str(attestation.get("ledger_reviewer", "")).strip()
        submitted_by = str(payload.get("submitted_by", "")).strip()
        require_real_text(errors, ledger_reviewer, "attestation.ledger_reviewer")
        add_error(
            errors,
            bool(SUBMITTED_AT_RE.match(str(attestation.get("ledger_reviewed_at", "")).strip())),
            "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
        )
        add_error(
            errors,
            bool(ledger_reviewer and submitted_by and ledger_reviewer.casefold() != submitted_by.casefold()),
            "attestation.ledger_reviewer must be different from submitted_by",
        )
        submitted_at = parse_audit_timestamp(payload.get("submitted_at"))
        ledger_reviewed_at = parse_audit_timestamp(attestation.get("ledger_reviewed_at"))
        add_error(
            errors,
            bool(submitted_at and ledger_reviewed_at and ledger_reviewed_at >= submitted_at),
            "attestation.ledger_reviewed_at must be at or after submitted_at",
        )
    validate_real_artifact_payloads(payload, errors, root, template_expected)
    return {
        "path": rel_path(path, root),
        "evidence_key": evidence_key,
        "status": "pass" if not errors else "fail",
        "template_only": template_expected,
        "artifact_integrity": artifact_integrity,
        "errors": errors,
    }