yao-meta-skill/scripts/create_iteration_snapshot.py

#!/usr/bin/env python3
import argparse
import json
import re
from datetime import date
from pathlib import Path


ROOT = Path(__file__).resolve().parent.parent

OPTIMIZATION_REPORTS = {
    "yao-meta-skill": ROOT / "reports" / "description_optimization.json",
    "team-frontend-review": ROOT / "examples" / "team-frontend-review" / "optimization" / "reports" / "description_optimization.json",
    "governed-incident-command": ROOT
    / "examples"
    / "governed-incident-command"
    / "optimization"
    / "reports"
    / "description_optimization.json",
}

CONTEXT_REPORTS = {
    "yao-meta-skill": ROOT / "reports" / "context_budget.json",
    "team-frontend-review": ROOT
    / "examples"
    / "team-frontend-review"
    / "generated-skill"
    / "reports"
    / "context_budget.json",
    "governed-incident-command": ROOT
    / "examples"
    / "governed-incident-command"
    / "generated-skill"
    / "reports"
    / "context_budget.json",
}


def load_json(path: Path) -> dict:
    return json.loads(path.read_text(encoding="utf-8"))


def relative(path: Path) -> str:
    return str(path.relative_to(ROOT))


def slugify(text: str) -> str:
    value = re.sub(r"[^a-z0-9]+", "-", text.strip().lower())
    return value.strip("-") or "snapshot"


def summarize_target(target: dict, route_scorecard: dict, optimization_report: dict, context_report: dict) -> dict:
    route_stats = route_scorecard.get("route_stats", {}).get(target["name"], {})
    winner_summary = optimization_report.get("winner", {})
    holdout = winner_summary.get("holdout", {})
    blind = winner_summary.get("blind_holdout", {})
    judge_blind = ((target.get("judge_blind") or {}).get("winner") or {})
    adversarial = winner_summary.get("adversarial_holdout", {})
    adversarial_calibration = ((target.get("calibration") or {}).get("adversarial_holdout") or {})
    return {
        "winner_label": target["winner_label"],
        "decision": target["decision"],
        "visible_holdout_total_errors": holdout.get("false_positives", 0) + holdout.get("false_negatives", 0),
        "blind_holdout_total_errors": blind.get("false_positives", 0) + blind.get("false_negatives", 0),
        "judge_blind_total_errors": target.get("winner_judge_blind_holdout_total_errors"),
        "judge_blind_agreement": judge_blind.get("agreement_rate"),
        "adversarial_total_errors": adversarial.get("false_positives", 0) + adversarial.get("false_negatives", 0),
        "adversarial_gap": adversarial_calibration.get("score_gap"),
        "adversarial_risk_band": adversarial_calibration.get("risk_band"),
        "route_precision": route_stats.get("precision"),
        "route_recall": route_stats.get("recall"),
        "route_average_margin": route_stats.get("average_margin"),
        "context_initial_load_tokens": context_report.get("initial_load_tokens"),
        "context_budget": context_report.get("budget"),
        "quality_density": context_report.get("quality_density"),
    }


def render_markdown(snapshot: dict) -> str:
    summary = snapshot["summary"]
    lines = [
        f"# Release Snapshot: {snapshot['target']}",
        "",
        f"- label: `{snapshot['label']}`",
        f"- date: `{snapshot['date']}`",
        f"- decision: `{snapshot['decision']}`",
        f"- winner label: `{summary['winner_label']}`",
        f"- next action: {snapshot['next_action']}",
        "",
        "## Promotion Gates",
        "",
        "| Gate | Pass |",
        "| --- | --- |",
    ]
    for gate, passed in snapshot["gate_status"].items():
        lines.append(f"| `{gate}` | {passed} |")
    lines.extend(
        [
            "",
            "## Quality Summary",
            "",
            f"- visible holdout total errors: `{summary['visible_holdout_total_errors']}`",
            f"- blind holdout total errors: `{summary['blind_holdout_total_errors']}`",
            f"- judge blind total errors: `{summary['judge_blind_total_errors']}`",
            f"- judge blind agreement: `{summary['judge_blind_agreement']}`",
            f"- adversarial total errors: `{summary['adversarial_total_errors']}`",
            f"- adversarial gap: `{summary['adversarial_gap']}`",
            f"- adversarial risk band: `{summary['adversarial_risk_band']}`",
            f"- route precision: `{summary['route_precision']}`",
            f"- route recall: `{summary['route_recall']}`",
            f"- route average margin: `{summary['route_average_margin']}`",
            f"- context initial load: `{summary['context_initial_load_tokens']}/{summary['context_budget']}`",
            f"- quality density: `{summary['quality_density']}`",
            "",
            "## Cause Tags",
            "",
        ]
    )
    for cause in snapshot["cause_tags"]:
        lines.append(f"- `{cause}`")
    lines.extend(["", "## Artifact Paths", ""])
    for key, path in snapshot["artifacts"].items():
        lines.append(f"- {key}: `{path}`")
    return "\n".join(lines) + "\n"


def main() -> None:
    parser = argparse.ArgumentParser(description="Create a versioned release snapshot from current promotion outputs.")
    parser.add_argument("--target", required=True)
    parser.add_argument("--label", default="manual")
    parser.add_argument("--promotion-decisions", default="reports/promotion_decisions.json")
    parser.add_argument("--route-scorecard", default="reports/route_scorecard.json")
    parser.add_argument("--output-dir", default="reports/release_snapshots")
    args = parser.parse_args()

    promotion_decisions = load_json(ROOT / args.promotion_decisions)
    route_scorecard = load_json(ROOT / args.route_scorecard)
    target = next((item for item in promotion_decisions["targets"] if item["name"] == args.target), None)
    if target is None:
        raise SystemExit(f"Unknown promotion target: {args.target}")

    optimization_report = load_json(OPTIMIZATION_REPORTS[args.target])
    context_report = load_json(CONTEXT_REPORTS[args.target])
    label = f"{date.today().isoformat()}-{slugify(args.label)}"
    output_dir = ROOT / args.output_dir / args.target
    output_dir.mkdir(parents=True, exist_ok=True)
    snapshot_json = output_dir / f"{label}.json"
    snapshot_md = output_dir / f"{label}.md"

    bundle_dir = ROOT / "reports" / "iteration_bundles" / args.target
    summary = summarize_target(target, route_scorecard, optimization_report, context_report)
    snapshot = {
        "target": args.target,
        "label": label,
        "date": date.today().isoformat(),
        "decision": target["decision"],
        "next_action": target["promotion"]["next_action"],
        "cause_tags": target["promotion"]["causes"],
        "gate_status": target["promotion"]["gate_status"],
        "summary": summary,
        "artifacts": {
            "bundle_json": relative(bundle_dir / "bundle.json"),
            "bundle_md": relative(bundle_dir / "bundle.md"),
            "review_md": relative(bundle_dir / "review.md"),
            "optimization_report": relative(OPTIMIZATION_REPORTS[args.target]),
            "promotion_decisions": relative(ROOT / args.promotion_decisions),
            "route_scorecard": relative(ROOT / args.route_scorecard),
            "context_budget": relative(CONTEXT_REPORTS[args.target]),
            "iteration_ledger": "reports/iteration_ledger.md",
        },
    }

    snapshot_json.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    snapshot_md.write_text(render_markdown(snapshot), encoding="utf-8")
    print(
        json.dumps(
            {
                "ok": True,
                "target": args.target,
                "label": label,
                "artifacts": {
                    "snapshot_json": relative(snapshot_json),
                    "snapshot_md": relative(snapshot_md),
                    "review_md": relative(bundle_dir / "review.md"),
                    "bundle_md": relative(bundle_dir / "bundle.md"),
                },
            },
            ensure_ascii=False,
            indent=2,
        )
    )


if __name__ == "__main__":
    main()