#!/usr/bin/env python3 import argparse import json from datetime import date from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parent.parent SCRIPT_INTERFACE = "cli" SCRIPT_INTERFACE_REASON = "Renders a cross-report evidence consistency gate for generated Skill OS reports." REQUIRED_REPORTS = { "benchmark": "reports/benchmark_reproducibility.json", "overview": "reports/skill-overview.json", "interpretation": "reports/skill-interpretation.json", "adoption": "reports/adoption_drift_report.json", "world_class_ledger": "reports/world_class_evidence_ledger.json", "skill_os2_coverage": "reports/skill_os2_coverage.json", "review_studio": "reports/review-studio.json", } BENCHMARK_SUMMARY_KEYS = [ "release_lock_ready", "required_artifact_count", "missing_artifact_count", "source_contract_sha256", "archive_sha256", "world_class_ledger_pending_count", "world_class_source_check_count", "world_class_source_pass_count", "world_class_source_blocked_count", "public_claim_ready", "public_claim_blocker_count", ] ADOPTION_SUMMARY_KEYS = [ "event_count", "adoption_sample_count", "activation_count", "accepted_count", "adoption_rate", "risk_band", "event_types", "source_types", ] LEDGER_SUMMARY_KEYS = [ "ledger_entry_count", "accepted_count", "pending_count", "human_pending_count", "external_pending_count", "source_check_count", "source_pass_count", "source_blocked_count", "ready_to_claim_world_class", "decision", ] LOCKSTEP_SECTIONS = [ "scorecard", "capability_profile", "principle_model", "contract_boundary", "quality_review", "risk_governance", "world_class_readiness", "package_assets", "iteration_roadmap", ] def load_json(path: Path) -> tuple[dict[str, Any], str | None]: if not path.exists(): return {}, "missing" try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: return {}, f"invalid-json: {exc}" if not isinstance(payload, dict): return {}, "json-root-not-object" return payload, None def rel_path(path: Path, root: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) except ValueError: return str(path.resolve()) def nested(payload: dict[str, Any], path: list[str], default: Any = None) -> Any: current: Any = payload for key in path: if not isinstance(current, dict) or key not in current: return default current = current[key] return current def add_check( checks: list[dict[str, Any]], *, key: str, label: str, status: str, expected: Any, actual: Any, paths: list[str], detail: str, ) -> None: checks.append( { "key": key, "label": label, "status": status, "expected": expected, "actual": actual, "paths": paths, "detail": detail, } ) def compare_values( checks: list[dict[str, Any]], *, key: str, label: str, expected: Any, actual: Any, paths: list[str], detail: str, ) -> None: add_check( checks, key=key, label=label, status="pass" if expected == actual else "fail", expected=expected, actual=actual, paths=paths, detail=detail, ) def compare_summary_keys( checks: list[dict[str, Any]], *, key_prefix: str, label: str, source_summary: dict[str, Any], embedded_summary: dict[str, Any], keys: list[str], paths: list[str], ) -> None: expected = {key: source_summary.get(key) for key in keys} actual = {key: embedded_summary.get(key) for key in keys} compare_values( checks, key=key_prefix, label=label, expected=expected, actual=actual, paths=paths, detail="Selected summary fields must match exactly across generated reports.", ) def report_contract(payload: dict[str, Any]) -> dict[str, Any]: contract = payload.get("report_contract") return contract if isinstance(contract, dict) else {} def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: reports: dict[str, dict[str, Any]] = {} checks: list[dict[str, Any]] = [] load_failures: dict[str, str] = {} for name, relative in REQUIRED_REPORTS.items(): payload, failure = load_json(skill_dir / relative) reports[name] = payload if failure: load_failures[relative] = failure add_check( checks, key="required-report-artifacts", label="Required report artifacts are readable", status="pass" if not load_failures else "fail", expected="all required JSON reports exist and parse", actual=load_failures or "all readable", paths=list(REQUIRED_REPORTS.values()), detail="The consistency gate can only be trusted when every source report is present and valid JSON.", ) benchmark = reports["benchmark"] overview = reports["overview"] interpretation = reports["interpretation"] adoption = reports["adoption"] ledger = reports["world_class_ledger"] coverage = reports["skill_os2_coverage"] review_studio = reports["review_studio"] benchmark_summary = nested(benchmark, ["summary"], {}) adoption_summary = nested(adoption, ["summary"], {}) ledger_summary = nested(ledger, ["summary"], {}) coverage_summary = nested(coverage, ["summary"], {}) studio_summary = nested(review_studio, ["summary"], {}) if isinstance(benchmark_summary, dict): compare_values( checks, key="benchmark-release-lock-self-consistency", label="Benchmark release lock matches git dirty state", expected=not bool(nested(benchmark, ["git_status", "dirty"], True)), actual=benchmark_summary.get("release_lock_ready"), paths=[REQUIRED_REPORTS["benchmark"]], detail="The benchmark release lock must reflect the generation-time git dirty flag.", ) for report_key, payload in [("overview", overview), ("interpretation", interpretation)]: embedded_benchmark = nested(payload, ["benchmark_reproducibility"], {}) compare_values( checks, key=f"{report_key}-benchmark-commit", label=f"{report_key} embeds the benchmark commit", expected=benchmark.get("commit"), actual=nested(embedded_benchmark, ["commit"]), paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], detail="Human-facing reports must point to the same benchmark release-lock commit.", ) compare_summary_keys( checks, key_prefix=f"{report_key}-benchmark-summary", label=f"{report_key} embeds benchmark summary fields", source_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {}, embedded_summary=nested(embedded_benchmark, ["summary"], {}), keys=BENCHMARK_SUMMARY_KEYS, paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], ) compare_summary_keys( checks, key_prefix=f"{report_key}-adoption-summary", label=f"{report_key} embeds adoption drift summary fields", source_summary=adoption_summary if isinstance(adoption_summary, dict) else {}, embedded_summary=nested(payload, ["adoption_drift", "summary"], {}), keys=ADOPTION_SUMMARY_KEYS, paths=[REQUIRED_REPORTS["adoption"], REQUIRED_REPORTS[report_key]], ) compare_summary_keys( checks, key_prefix=f"{report_key}-world-class-ledger-summary", label=f"{report_key} embeds world-class ledger summary fields", source_summary=ledger_summary if isinstance(ledger_summary, dict) else {}, embedded_summary=nested(payload, ["world_class_evidence_ledger", "summary"], {}), keys=LEDGER_SUMMARY_KEYS, paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], ) readiness_expected = { "ready": ledger_summary.get("ready_to_claim_world_class") if isinstance(ledger_summary, dict) else None, "decision": ledger_summary.get("decision") if isinstance(ledger_summary, dict) else None, "pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, "accepted_count": ledger_summary.get("accepted_count") if isinstance(ledger_summary, dict) else None, "source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None, "source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None, } readiness = nested(payload, ["world_class_readiness"], {}) readiness_actual = {key: readiness.get(key) if isinstance(readiness, dict) else None for key in readiness_expected} compare_values( checks, key=f"{report_key}-world-class-readiness", label=f"{report_key} derives readiness from the ledger", expected=readiness_expected, actual=readiness_actual, paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], detail="Readiness summaries must be derived from the evidence ledger, not hand-maintained copy.", ) for section in LOCKSTEP_SECTIONS: compare_values( checks, key=f"overview-interpretation-lockstep-{section.replace('_', '-')}", label=f"Overview and interpretation share {section}", expected=overview.get(section), actual=interpretation.get(section), paths=[REQUIRED_REPORTS["overview"], REQUIRED_REPORTS["interpretation"]], detail="The first-class interpretation report must stay in lockstep with the canonical overview model.", ) for report_key, expected_html in [ ("overview", "reports/skill-overview.html"), ("interpretation", "reports/skill-interpretation.html"), ]: contract = report_contract(reports[report_key]) expected = { "schema_version": "2.0", "default_language": "zh-CN", "layout": "kami-white-audit-v2", "html_report": expected_html, "html_exists": True, } actual = { "schema_version": contract.get("schema_version"), "default_language": contract.get("default_language"), "layout": contract.get("layout"), "html_report": contract.get("html_report"), "html_exists": (skill_dir / expected_html).exists(), } compare_values( checks, key=f"{report_key}-html-contract", label=f"{report_key} has a stable HTML contract", expected=expected, actual=actual, paths=[REQUIRED_REPORTS[report_key], expected_html], detail="Report output paths and language defaults are part of the user-facing contract.", ) if isinstance(ledger_summary, dict): expected_boundary = { "world_class_evidence_pending_count": ledger_summary.get("pending_count"), "public_world_class_ready": ledger_summary.get("ready_to_claim_world_class"), } actual_boundary = { "world_class_evidence_pending_count": coverage_summary.get("world_class_evidence_pending_count") if isinstance(coverage_summary, dict) else None, "public_world_class_ready": coverage_summary.get("public_world_class_ready") if isinstance(coverage_summary, dict) else None, } compare_values( checks, key="coverage-world-class-boundary", label="Coverage report mirrors world-class evidence boundary", expected=expected_boundary, actual=actual_boundary, paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["skill_os2_coverage"]], detail="Blueprint coverage can be locally complete while public world-class evidence remains pending.", ) benchmark_boundary = { "world_class_ledger_pending_count": ledger_summary.get("pending_count"), "world_class_source_check_count": ledger_summary.get("source_check_count"), "world_class_source_pass_count": ledger_summary.get("source_pass_count"), "world_class_source_blocked_count": ledger_summary.get("source_blocked_count"), "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), } actual_benchmark_boundary = { key: benchmark_summary.get(key) if isinstance(benchmark_summary, dict) else None for key in benchmark_boundary } compare_values( checks, key="benchmark-world-class-boundary", label="Benchmark report mirrors world-class evidence boundary", expected=benchmark_boundary, actual=actual_benchmark_boundary, paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], detail="Benchmark reproducibility must not overstate public claim readiness.", ) public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False compare_values( checks, key="review-studio-no-overclaim", label="Review Studio does not overclaim pending world-class evidence", expected=False if not public_ready else True, actual=studio_summary.get("decision") in {"pass", "release", "ready", "world-class-ready"} if isinstance(studio_summary, dict) else None, paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["review_studio"]], detail="When world-class evidence is pending, Review Studio must stay in a review or warning posture.", ) status_counts: dict[str, int] = {"pass": 0, "warn": 0, "fail": 0} for check in checks: status_counts[check["status"]] = status_counts.get(check["status"], 0) + 1 summary = { "check_count": len(checks), "pass_count": status_counts.get("pass", 0), "warn_count": status_counts.get("warn", 0), "fail_count": status_counts.get("fail", 0), "decision": "consistent" if status_counts.get("fail", 0) == 0 else "evidence-drift-detected", } return { "schema_version": "1.0", "ok": summary["fail_count"] == 0, "generated_at": generated_at, "skill_dir": rel_path(skill_dir, ROOT), "summary": summary, "status_counts": status_counts, "checks": checks, "artifacts": { "json": "reports/evidence_consistency.json", "markdown": "reports/evidence_consistency.md", }, } def render_markdown(report: dict[str, Any]) -> str: summary = report["summary"] lines = [ "# Evidence Consistency", "", f"Generated at: `{report['generated_at']}`", "", "## Summary", "", f"- decision: `{summary['decision']}`", f"- checks: `{summary['check_count']}`", f"- pass: `{summary['pass_count']}`", f"- warn: `{summary['warn_count']}`", f"- fail: `{summary['fail_count']}`", "", "This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist.", "", "## Checks", "", "| Check | Status | Detail | Paths |", "| --- | --- | --- | --- |", ] for check in report["checks"]: paths = ", ".join(f"`{path}`" for path in check["paths"]) lines.append( "| " + " | ".join( [ check["label"].replace("|", "\\|"), f"`{check['status']}`", check["detail"].replace("|", "\\|"), paths.replace("|", "\\|"), ] ) + " |" ) failures = [check for check in report["checks"] if check["status"] == "fail"] if failures: lines.extend(["", "## Failures", ""]) for check in failures: lines.extend( [ f"### {check['label']}", "", f"- key: `{check['key']}`", f"- expected: `{json.dumps(check['expected'], ensure_ascii=False, sort_keys=True)}`", f"- actual: `{json.dumps(check['actual'], ensure_ascii=False, sort_keys=True)}`", "", ] ) return "\n".join(lines).rstrip() + "\n" def main() -> None: parser = argparse.ArgumentParser(description="Render cross-report evidence consistency checks.") parser.add_argument("skill_dir", nargs="?", default=".") parser.add_argument("--output-json", default="reports/evidence_consistency.json") parser.add_argument("--output-md", default="reports/evidence_consistency.md") parser.add_argument("--generated-at", default=date.today().isoformat()) args = parser.parse_args() skill_dir = Path(args.skill_dir).resolve() report = build_report(skill_dir, args.generated_at) output_json = Path(args.output_json) output_md = Path(args.output_md) if not output_json.is_absolute(): output_json = skill_dir / output_json if not output_md.is_absolute(): output_md = skill_dir / output_md output_json.parent.mkdir(parents=True, exist_ok=True) output_md.parent.mkdir(parents=True, exist_ok=True) output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") output_md.write_text(render_markdown(report), encoding="utf-8") print(json.dumps(report, ensure_ascii=False, indent=2)) raise SystemExit(0 if report["ok"] else 2) if __name__ == "__main__": main()