yao-meta-skill/scripts/render_world_class_evidence_intake.py

#!/usr/bin/env python3
import argparse
import json
from datetime import date
from pathlib import Path
from typing import Any

from render_world_class_evidence_ledger import build_ledger


ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Validates world-class human and external evidence intake packets before ledger review."

REQUIRED_TOP_LEVEL = [
    "schema_version",
    "evidence_key",
    "template_only",
    "category",
    "source_type",
    "submitted_by",
    "submitted_at",
    "summary",
    "artifact_refs",
    "provenance",
    "privacy",
    "anti_overclaim",
    "attestation",
]
REQUIRED_PRIVACY_FALSE = [
    "raw_user_content_included",
    "raw_provider_prompt_included",
    "credentials_included",
    "secrets_included",
]
REQUIRED_ANTI_OVERCLAIM_FALSE = [
    "planned_work_counts_as_evidence",
    "metadata_fallback_counts_as_native_enforcement",
    "pending_review_counts_as_human_decision",
    "local_command_runner_counts_as_provider_model",
]
REQUIRED_ATTESTATION_TRUE = [
    "real_external_or_human_evidence",
    "reviewer_or_operator_identity_present",
    "artifact_refs_reviewed",
    "privacy_contract_satisfied",
]
EXPECTED_SOURCE_TYPES = {
    "provider-holdout": "provider-output-eval",
    "human-adjudication": "blind-ab-review",
    "native-permission-enforcement": "runtime-permission-guard",
    "native-client-telemetry": "native-client-telemetry",
}


def load_json(path: Path) -> dict[str, Any]:
    if not path.exists():
        return {}
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return {}
    return payload if isinstance(payload, dict) else {}


def rel_path(path: Path, root: Path) -> str:
    try:
        return str(path.resolve().relative_to(root.resolve()))
    except ValueError:
        return str(path.resolve())


def add_error(errors: list[str], condition: bool, message: str) -> None:
    if not condition:
        errors.append(message)


def validate_artifact_refs(payload: dict[str, Any], errors: list[str]) -> None:
    refs = payload.get("artifact_refs")
    add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference")
    if not isinstance(refs, list):
        return
    for index, ref in enumerate(refs):
        if not isinstance(ref, dict):
            errors.append(f"artifact_refs[{index}] must be an object")
            continue
        add_error(errors, bool(str(ref.get("path", "")).strip()), f"artifact_refs[{index}].path is required")
        add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required")
        add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content")


def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None:
    key = str(payload.get("evidence_key", ""))
    provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
    if key == "provider-holdout":
        add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required")
        add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required")
        add_error(
            errors,
            provenance.get("credential_material_committed") is False,
            "provider-holdout must attest credential_material_committed is false",
        )
    elif key == "human-adjudication":
        add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
        add_error(
            errors,
            provenance.get("answer_key_opened_after_decisions") is True,
            "human-adjudication must attest answer_key_opened_after_decisions is true",
        )
    elif key == "native-permission-enforcement":
        add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
        add_error(
            errors,
            bool(str(provenance.get("guard_location", "")).strip()),
            "native-permission-enforcement provenance.guard_location is required",
        )
        add_error(
            errors,
            provenance.get("metadata_fallback_retained_for_other_targets") is True,
            "native-permission-enforcement must retain metadata fallback for non-native targets",
        )
    elif key == "native-client-telemetry":
        add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required")
        add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external")
        add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only")


def validate_payload(
    payload: dict[str, Any],
    entry: dict[str, Any],
    *,
    path: Path,
    root: Path,
    template_expected: bool,
) -> dict[str, Any]:
    errors: list[str] = []
    for key in REQUIRED_TOP_LEVEL:
        add_error(errors, key in payload, f"missing {key}")
    evidence_key = str(entry.get("key", ""))
    add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0")
    add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}")
    add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}")
    add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}")
    add_error(
        errors,
        payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key),
        f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}",
    )
    add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required")
    add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required")
    add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required")
    validate_artifact_refs(payload, errors)
    privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {}
    for key in REQUIRED_PRIVACY_FALSE:
        add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false")
    anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {}
    for key in REQUIRED_ANTI_OVERCLAIM_FALSE:
        add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false")
    validate_evidence_specific(payload, errors)
    attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
    if not template_expected:
        for key in REQUIRED_ATTESTATION_TRUE:
            add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission")
    return {
        "path": rel_path(path, root),
        "evidence_key": evidence_key,
        "status": "pass" if not errors else "fail",
        "template_only": template_expected,
        "errors": errors,
    }


def template_paths(skill_dir: Path, keys: list[str]) -> dict[str, Path]:
    template_dir = skill_dir / "evidence" / "world_class" / "templates"
    return {key: template_dir / f"{key}.intake.json" for key in keys}


def submission_paths(submissions_dir: Path) -> list[Path]:
    if not submissions_dir.exists():
        return []
    return sorted(path for path in submissions_dir.glob("*.json") if path.is_file())


def find_entry(entries: list[dict[str, Any]], key: str) -> dict[str, Any] | None:
    for entry in entries:
        if entry.get("key") == key:
            return entry
    return None


def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
    ledger = build_ledger(skill_dir, generated_at)
    entries = ledger.get("entries", [])
    keys = [str(entry.get("key", "")) for entry in entries]
    schema_path = skill_dir / "evidence" / "world_class" / "intake.schema.json"
    template_results = []
    for key, path in template_paths(skill_dir, keys).items():
        entry = find_entry(entries, key) or {"key": key, "category": "external"}
        payload = load_json(path)
        if not payload:
            template_results.append(
                {
                    "path": rel_path(path, skill_dir),
                    "evidence_key": key,
                    "status": "fail",
                    "template_only": True,
                    "errors": ["template missing or invalid JSON"],
                }
            )
            continue
        template_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=True))

    submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
    submission_results = []
    for path in submission_paths(submissions_dir):
        payload = load_json(path)
        key = str(payload.get("evidence_key", ""))
        entry = find_entry(entries, key)
        if not payload or entry is None:
            submission_results.append(
                {
                    "path": rel_path(path, skill_dir),
                    "evidence_key": key or "unknown",
                    "status": "fail",
                    "template_only": False,
                    "errors": ["submission missing, invalid JSON, or unknown evidence_key"],
                }
            )
            continue
        submission_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=False))

    template_pass_count = sum(1 for item in template_results if item["status"] == "pass")
    valid_submission_count = sum(1 for item in submission_results if item["status"] == "pass")
    invalid_submission_count = sum(1 for item in submission_results if item["status"] == "fail")
    schema_exists = schema_path.exists()
    intake_ready = schema_exists and template_pass_count == len(keys) and invalid_submission_count == 0
    return {
        "schema_version": "1.0",
        "ok": intake_ready,
        "generated_at": generated_at,
        "skill_dir": rel_path(skill_dir, ROOT),
        "summary": {
            "schema_present": schema_exists,
            "ledger_entry_count": len(keys),
            "template_count": len(template_results),
            "template_pass_count": template_pass_count,
            "submission_count": len(submission_results),
            "valid_submission_count": valid_submission_count,
            "invalid_submission_count": invalid_submission_count,
            "ready_for_external_collection": intake_ready,
            "ready_for_ledger_review": valid_submission_count > 0 and invalid_submission_count == 0,
            "ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True,
            "overclaim_guard_active": True,
            "decision": (
                "fix-intake"
                if not intake_ready
                else ("intake-ready-for-ledger-review" if valid_submission_count else "awaiting-submissions")
            ),
        },
        "schema": rel_path(schema_path, skill_dir),
        "templates": template_results,
        "submissions": submission_results,
        "source_ledger": {
            "json": "reports/world_class_evidence_ledger.json",
            "markdown": "reports/world_class_evidence_ledger.md",
            "pending_count": ledger.get("summary", {}).get("pending_count", 0),
        },
        "artifacts": {
            "json": "reports/world_class_evidence_intake.json",
            "markdown": "reports/world_class_evidence_intake.md",
        },
    }


def render_table(items: list[dict[str, Any]]) -> list[str]:
    lines = ["| Evidence | Status | Path | Errors |", "| --- | --- | --- | --- |"]
    if not items:
        lines.append("| `none` | `n/a` | none | none |")
        return lines
    for item in items:
        errors = "; ".join(item.get("errors", [])) or "none"
        safe_errors = errors.replace("|", "\\|")
        lines.append(f"| `{item['evidence_key']}` | `{item['status']}` | `{item['path']}` | {safe_errors} |")
    return lines


def render_markdown(report: dict[str, Any]) -> str:
    summary = report["summary"]
    lines = [
        "# World-Class Evidence Intake",
        "",
        f"Generated at: `{report['generated_at']}`",
        "",
        "## Summary",
        "",
        f"- decision: `{summary['decision']}`",
        f"- schema present: `{str(summary['schema_present']).lower()}`",
        f"- templates: `{summary['template_pass_count']}` / `{summary['template_count']}`",
        f"- submissions: `{summary['valid_submission_count']}` valid / `{summary['submission_count']}` total",
        f"- invalid submissions: `{summary['invalid_submission_count']}`",
        f"- ready for external collection: `{str(summary['ready_for_external_collection']).lower()}`",
        f"- ready for ledger review: `{str(summary['ready_for_ledger_review']).lower()}`",
        f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
        f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
        "",
        "This report validates the intake contract for human and external evidence. A valid intake packet means the evidence is ready for ledger review; it does not by itself make a world-class claim true.",
        "",
        "## Templates",
        "",
        *render_table(report["templates"]),
        "",
        "## Submissions",
        "",
        *render_table(report["submissions"]),
        "",
        "## Boundary",
        "",
        "- Templates and planned work do not count as accepted evidence.",
        "- Local command-runner output does not count as provider-backed model evidence.",
        "- Metadata fallback does not count as native permission enforcement.",
        "- Pending reviewer work does not count as human adjudication.",
    ]
    return "\n".join(lines).rstrip() + "\n"


def main() -> None:
    parser = argparse.ArgumentParser(description="Validate world-class evidence intake packets.")
    parser.add_argument("skill_dir", nargs="?", default=".")
    parser.add_argument("--submissions-dir")
    parser.add_argument("--output-json", default="reports/world_class_evidence_intake.json")
    parser.add_argument("--output-md", default="reports/world_class_evidence_intake.md")
    parser.add_argument("--generated-at", default=date.today().isoformat())
    args = parser.parse_args()

    skill_dir = Path(args.skill_dir).resolve()
    submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
    report = build_intake(skill_dir, args.generated_at, submissions_dir)
    output_json = Path(args.output_json)
    output_md = Path(args.output_md)
    if not output_json.is_absolute():
        output_json = skill_dir / output_json
    if not output_md.is_absolute():
        output_md = skill_dir / output_md
    output_json.parent.mkdir(parents=True, exist_ok=True)
    output_md.parent.mkdir(parents=True, exist_ok=True)
    output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    output_md.write_text(render_markdown(report), encoding="utf-8")
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()