#!/usr/bin/env python3 """Shared helpers for cross-report evidence consistency checks.""" import json from pathlib import Path from typing import Any SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers." REQUIRED_REPORTS = { "benchmark": "reports/benchmark_reproducibility.json", "overview": "reports/skill-overview.json", "interpretation": "reports/skill-interpretation.json", "adoption": "reports/adoption_drift_report.json", "world_class_ledger": "reports/world_class_evidence_ledger.json", "world_class_plan": "reports/world_class_evidence_plan.json", "world_class_intake": "reports/world_class_evidence_intake.json", "world_class_preflight": "reports/world_class_evidence_preflight.json", "world_class_submission_review": "reports/world_class_submission_review.json", "world_class_operator_runbook": "reports/world_class_operator_runbook.json", "skill_os2_coverage": "reports/skill_os2_coverage.json", "review_studio": "reports/review-studio.json", "package_verification": "reports/package_verification.json", "install_simulation": "reports/install_simulation.json", "trust": "reports/security_trust_report.json", "context_budget": "reports/context_budget.json", "world_class_claim_guard": "reports/world_class_claim_guard.json", } REQUIRED_TEXT_REPORTS = { "skill_os2_review": "reports/skill-os-2-review.md", } BENCHMARK_SUMMARY_KEYS = [ "release_lock_ready", "required_artifact_count", "missing_artifact_count", "source_contract_sha256", "archive_sha256", "world_class_ledger_pending_count", "world_class_source_check_count", "world_class_source_pass_count", "world_class_source_blocked_count", "public_claim_ready", "public_claim_blocker_count", ] ADOPTION_SUMMARY_KEYS = [ "event_count", "adoption_sample_count", "activation_count", "accepted_count", "adoption_rate", "risk_band", "event_types", "source_types", ] LEDGER_SUMMARY_KEYS = [ "ledger_entry_count", "accepted_count", "pending_count", "human_pending_count", "external_pending_count", "source_check_count", "source_pass_count", "source_blocked_count", "ready_to_claim_world_class", "decision", ] LOCKSTEP_SECTIONS = [ "scorecard", "capability_profile", "principle_model", "contract_boundary", "quality_review", "risk_governance", "world_class_readiness", "package_assets", "iteration_roadmap", ] def load_json(path: Path) -> tuple[dict[str, Any], str | None]: if not path.exists(): return {}, "missing" try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return {}, f"invalid-json: {exc}" if not isinstance(payload, dict): return {}, "json-root-not-object" return payload, None def load_text(path: Path) -> tuple[str, str | None]: if not path.exists(): return "", "missing" return path.read_text(encoding="utf-8"), None def rel_path(path: Path, root: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) except ValueError: return str(path.resolve()) def nested(payload: dict[str, Any], path: list[str], default: Any = None) -> Any: current: Any = payload for key in path: if not isinstance(current, dict) or key not in current: return default current = current[key] return current def scanned_surface_paths(payload: dict[str, Any]) -> set[str]: surfaces = payload.get("scanned_surfaces") if not isinstance(surfaces, list): return set() paths: set[str] = set() for item in surfaces: if isinstance(item, dict) and isinstance(item.get("path"), str): paths.add(item["path"]) elif isinstance(item, str): paths.add(item) return paths def as_int(value: Any) -> int | None: if isinstance(value, bool): return None try: return int(value) except (TypeError, ValueError): return None def add_check( checks: list[dict[str, Any]], *, key: str, label: str, status: str, expected: Any, actual: Any, paths: list[str], detail: str, ) -> None: checks.append( { "key": key, "label": label, "status": status, "expected": expected, "actual": actual, "paths": paths, "detail": detail, } ) def compare_values( checks: list[dict[str, Any]], *, key: str, label: str, expected: Any, actual: Any, paths: list[str], detail: str, ) -> None: add_check( checks, key=key, label=label, status="pass" if expected == actual else "fail", expected=expected, actual=actual, paths=paths, detail=detail, ) def compare_summary_keys( checks: list[dict[str, Any]], *, key_prefix: str, label: str, source_summary: dict[str, Any], embedded_summary: dict[str, Any], keys: list[str], paths: list[str], ) -> None: expected = {key: source_summary.get(key) for key in keys} actual = {key: embedded_summary.get(key) for key in keys} compare_values( checks, key=key_prefix, label=label, expected=expected, actual=actual, paths=paths, detail="Selected summary fields must match exactly across generated reports.", ) def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]: gates = review_studio.get("gates") if not isinstance(gates, list): return {} for item in gates: if isinstance(item, dict) and item.get("key") == key: return item return {} def report_contract(payload: dict[str, Any]) -> dict[str, Any]: contract = payload.get("report_contract") return contract if isinstance(contract, dict) else {} def render_markdown(report: dict[str, Any]) -> str: summary = report["summary"] lines = [ "# Evidence Consistency", "", f"Generated at: `{report['generated_at']}`", "", "## Summary", "", f"- decision: `{summary['decision']}`", f"- checks: `{summary['check_count']}`", f"- pass: `{summary['pass_count']}`", f"- warn: `{summary['warn_count']}`", f"- fail: `{summary['fail_count']}`", "", "This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist.", "", "## Checks", "", "| Check | Status | Detail | Paths |", "| --- | --- | --- | --- |", ] for check in report["checks"]: paths = ", ".join(f"`{path}`" for path in check["paths"]) lines.append( "| " + " | ".join( [ check["label"].replace("|", "\\|"), f"`{check['status']}`", check["detail"].replace("|", "\\|"), paths.replace("|", "\\|"), ] ) + " |" ) failures = [check for check in report["checks"] if check["status"] == "fail"] if failures: lines.extend(["", "## Failures", ""]) for check in failures: lines.extend( [ f"### {check['label']}", "", f"- key: `{check['key']}`", f"- expected: `{json.dumps(check['expected'], ensure_ascii=False, sort_keys=True)}`", f"- actual: `{json.dumps(check['actual'], ensure_ascii=False, sort_keys=True)}`", "", ] ) return "\n".join(lines).rstrip() + "\n"