#!/usr/bin/env python3 import argparse import json from pathlib import Path ROOT = Path(__file__).resolve().parent.parent OPTIMIZATION_REPORTS = { "yao-meta-skill": ROOT / "reports" / "description_optimization.json", "team-frontend-review": ROOT / "examples" / "team-frontend-review" / "optimization" / "reports" / "description_optimization.json", "governed-incident-command": ROOT / "examples" / "governed-incident-command" / "optimization" / "reports" / "description_optimization.json", } SKILL_PATHS = { "yao-meta-skill": ROOT / "SKILL.md", "team-frontend-review": ROOT / "examples" / "team-frontend-review" / "generated-skill" / "SKILL.md", "governed-incident-command": ROOT / "examples" / "governed-incident-command" / "generated-skill" / "SKILL.md", } def load_json(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) def error_total(prefix: str, target: dict) -> int: return int(target.get(f"{prefix}_fp", 0)) + int(target.get(f"{prefix}_fn", 0)) def family_clean(summary: dict | None) -> bool: if not summary: return False return summary.get("clean_family_count", 0) == summary.get("family_count", 0) def relative(path: Path) -> str: return str(path.relative_to(ROOT)) def classify_target(target: dict, route_scorecard: dict) -> dict: route_summary = route_scorecard["summary"] route_stats = route_scorecard["route_stats"].get(target["name"], {}) winner_changed = target["winner_label"] != "Current" visible_ok = error_total("winner_holdout", target) <= error_total("current_holdout", target) and error_total( "winner_holdout", target ) <= error_total("baseline_holdout", target) blind_ok = error_total("winner_blind_holdout", target) <= error_total( "current_blind_holdout", target ) and error_total("winner_blind_holdout", target) <= error_total("baseline_blind_holdout", target) judge_ok = error_total("winner_judge_blind_holdout", target) <= error_total( "current_judge_blind_holdout", target ) and error_total("winner_judge_blind_holdout", target) <= error_total("baseline_judge_blind_holdout", target) judge_agreement = ((target.get("judge_blind") or {}).get("winner") or {}).get("agreement_rate") judge_agreement_ok = judge_agreement is not None and judge_agreement >= 0.95 adversarial_ok = error_total("winner_adversarial_holdout", target) <= error_total( "current_adversarial_holdout", target ) and error_total("winner_adversarial_holdout", target) <= error_total("baseline_adversarial_holdout", target) holdout_calibration = (target.get("calibration") or {}).get("holdout") or {} adversarial_calibration = (target.get("calibration") or {}).get("adversarial_holdout") or {} adversarial_risk = adversarial_calibration.get("risk_band") adversarial_risk_ok = adversarial_risk not in {"overlap", None} route_clean = ( route_summary.get("misroute_count", 0) == 0 and route_summary.get("ambiguous_case_count", 0) == 0 and route_stats.get("precision") == 1.0 and route_stats.get("recall") == 1.0 ) family_ok = all( family_clean((target.get("family_health") or {}).get(gate)) for gate in ("blind_holdout", "judge_blind_holdout", "adversarial_holdout") ) causes = [] if not winner_changed: causes.append("no_candidate_outperformed_current") if not visible_ok: causes.append("visible_holdout_regression") if not blind_ok: causes.append("blind_holdout_regression") if error_total("winner_holdout", target) > 0: causes.append("current_holdout_gap_present") if holdout_calibration.get("risk_band") in {"overlap", "watch", "tight"}: causes.append("current_holdout_risk") if not judge_ok: causes.append("judge_blind_regression") if not judge_agreement_ok: causes.append("judge_blind_low_agreement") if not adversarial_ok: causes.append("adversarial_regression") if adversarial_risk == "overlap": causes.append("adversarial_overlap_risk") elif adversarial_risk in {"watch", "tight"}: causes.append("adversarial_watch_risk") if not family_ok: causes.append("family_instability") if route_summary.get("misroute_count", 0): causes.append("route_confusion") if route_summary.get("ambiguous_case_count", 0): causes.append("route_ambiguity") if target["winner_tokens"] > target["current_tokens"] and not winner_changed: causes.append("longer_without_gain") severe = ( route_summary.get("misroute_count", 0) > 0 or route_summary.get("ambiguous_case_count", 0) > 0 or adversarial_risk == "overlap" or not judge_agreement_ok ) all_gates_pass = all( [visible_ok, blind_ok, judge_ok, judge_agreement_ok, adversarial_ok, adversarial_risk_ok, route_clean, family_ok] ) if winner_changed and all_gates_pass: decision = "promote" elif severe: decision = "blocked" else: decision = "keep_current" if decision == "promote" and "promotion_ready" not in causes: causes.append("promotion_ready") next_action = { "promote": "Review the promoted candidate, update the skill description, and snapshot the release.", "keep_current": "Keep the current description and open a new candidate only when fresh route evidence appears.", "blocked": "Do not promote. Investigate failing gates, route collisions, or calibration risk before generating new candidates.", }[decision] return { "decision": decision, "winner_changed": winner_changed, "all_gates_pass": all_gates_pass, "route_clean": route_clean, "judge_agreement_ok": judge_agreement_ok, "causes": causes, "next_action": next_action, "gate_status": { "visible_holdout_non_regression": visible_ok, "blind_holdout_non_regression": blind_ok, "judge_blind_non_regression": judge_ok, "judge_blind_agreement": judge_agreement_ok, "adversarial_non_regression": adversarial_ok, "adversarial_risk_ok": adversarial_risk_ok, "route_confusion_clean": route_clean, "family_stability": family_ok, }, "route_metrics": { "precision": route_stats.get("precision"), "recall": route_stats.get("recall"), "average_margin": route_stats.get("average_margin"), "misroute_count": route_summary.get("misroute_count"), "ambiguous_case_count": route_summary.get("ambiguous_case_count"), }, } def build_candidate_entries(target_name: str, target_summary: dict, decision: str) -> list[dict]: report = load_json(OPTIMIZATION_REPORTS[target_name]) winner_description = report["winner"]["description"] winner_dev_errors = report["winner"]["dev"]["false_positives"] + report["winner"]["dev"]["false_negatives"] entries = [ { "name": target_name, "role": "baseline", "label": "Baseline", "ranking_state": "reference", "promotion_state": "reference", "description": report["baseline"]["description"] if report.get("baseline") else None, "tokens": report["baseline"]["estimated_tokens"] if report.get("baseline") else None, "dev_errors": ( report["baseline"]["dev"]["false_positives"] + report["baseline"]["dev"]["false_negatives"] if report.get("baseline") else None ), "holdout_errors": ( report["baseline"]["holdout"]["false_positives"] + report["baseline"]["holdout"]["false_negatives"] if report.get("baseline") and report["baseline"].get("holdout") else None ), } ] for candidate in report["candidates"]: selected = candidate["description"] == winner_description role = "current" if candidate["id"] == "current" else "candidate" if selected and decision == "promote": promotion_state = "promoted" elif selected and role == "current": promotion_state = "kept_current" elif selected: promotion_state = "blocked" else: promotion_state = "blocked" reason_tags = [] candidate_dev_errors = candidate["dev"]["false_positives"] + candidate["dev"]["false_negatives"] if candidate_dev_errors > winner_dev_errors: reason_tags.append("weaker_dev_fit") holdout = candidate.get("holdout") or {} holdout_errors = holdout.get("false_positives", 0) + holdout.get("false_negatives", 0) if holdout else None if holdout_errors is not None and holdout_errors > error_total("current_holdout", target_summary): reason_tags.append("visible_holdout_regression") if candidate["estimated_tokens"] > target_summary["current_tokens"] and not selected: reason_tags.append("longer_without_gain") if not reason_tags and not selected: reason_tags.append("not_selected_by_dev_ranking") entries.append( { "name": target_name, "role": role, "label": candidate["label"], "ranking_state": "selected_by_dev" if selected else "not_selected", "promotion_state": promotion_state, "description": candidate["description"], "tokens": candidate["estimated_tokens"], "dev_errors": candidate_dev_errors, "holdout_errors": holdout_errors, "reason_tags": reason_tags, } ) return entries def render_candidate_registry_md(targets: list[dict]) -> str: lines = [ "# Candidate Registry", "", "| Target | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | Reason Tags |", "| --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |", ] for target in targets: for entry in target["registry"]: lines.append( f"| `{target['name']}` | {entry['role']} | `{entry['label']}` | {entry['ranking_state']} | {entry['promotion_state']} | {'-' if entry['tokens'] is None else entry['tokens']} | {'-' if entry['dev_errors'] is None else entry['dev_errors']} | {'-' if entry['holdout_errors'] is None else entry['holdout_errors']} | {', '.join(entry.get('reason_tags', [])) or '-'} |" ) return "\n".join(lines) + "\n" def render_promotion_md(targets: list[dict]) -> str: lines = [ "# Promotion Decisions", "", "| Target | Decision | Winner | Route Clean | Judge Agreement | Adv Risk | Causes | Next Action |", "| --- | --- | --- | --- | --- | --- | --- | --- |", ] for target in targets: decision = target["decision"] judge_summary = ((target.get("judge_blind") or {}).get("winner") or {}).get("agreement_rate") adv_risk = ((target.get("calibration") or {}).get("adversarial_holdout") or {}).get("risk_band") lines.append( f"| `{target['name']}` | `{decision}` | `{target['winner_label']}` | {target['promotion']['route_clean']} | {judge_summary if judge_summary is not None else '-'} | {adv_risk or '-'} | {', '.join(target['promotion']['causes'])} | {target['promotion']['next_action']} |" ) return "\n".join(lines) + "\n" def render_bundle_md(bundle: dict) -> str: review = bundle["review_template"] lines = [ f"# Iteration Bundle: {bundle['target']}", "", f"- decision: `{bundle['decision']}`", f"- winner label: `{bundle['winner_label']}`", f"- winner changed: `{bundle['winner_changed']}`", f"- next action: {bundle['next_action']}", "", "## Cause Tags", "", ] for cause in bundle["cause_tags"]: lines.append(f"- `{cause}`") lines.extend( [ "", "## Gate Status", "", "| Gate | Pass |", "| --- | --- |", ] ) for gate, passed in bundle["gate_status"].items(): lines.append(f"| `{gate}` | {passed} |") lines.extend( [ "", "## Candidate Registry", "", "| Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors |", "| --- | --- | --- | --- | ---: | ---: | ---: |", ] ) for entry in bundle["candidate_registry"]: lines.append( f"| {entry['role']} | `{entry['label']}` | {entry['ranking_state']} | {entry['promotion_state']} | {'-' if entry['tokens'] is None else entry['tokens']} | {'-' if entry['dev_errors'] is None else entry['dev_errors']} | {'-' if entry['holdout_errors'] is None else entry['holdout_errors']} |" ) lines.extend( [ "", "## Human Review Stub", "", f"- target: {review['target']}", f"- current description: {review['current_description']}", f"- candidate description: {review['candidate_description']}", f"- review focus: {', '.join(review['focus'])}", "", "## Artifact Paths", "", ] ) for label, path in bundle["artifacts"].items(): lines.append(f"- {label}: `{path}`") return "\n".join(lines) + "\n" def main() -> None: parser = argparse.ArgumentParser(description="Apply promotion policy and build iteration bundles.") parser.add_argument("--optimization-suite", default="reports/description_optimization_suite.json") parser.add_argument("--route-scorecard", default="reports/route_scorecard.json") parser.add_argument("--output-json", default="reports/promotion_decisions.json") parser.add_argument("--output-md", default="reports/promotion_decisions.md") parser.add_argument("--candidate-registry-json", default="reports/candidate_registry.json") parser.add_argument("--candidate-registry-md", default="reports/candidate_registry.md") parser.add_argument("--bundle-dir", default="reports/iteration_bundles") args = parser.parse_args() suite = load_json(ROOT / args.optimization_suite) route_scorecard = load_json(ROOT / args.route_scorecard) bundle_dir = ROOT / args.bundle_dir bundle_dir.mkdir(parents=True, exist_ok=True) targets = [] decisions = [] for target in suite["targets"]: promotion = classify_target(target, route_scorecard) registry = build_candidate_entries(target["name"], target, promotion["decision"]) target["promotion"] = promotion target["registry"] = registry target["decision"] = promotion["decision"] targets.append(target) decisions.append(promotion["decision"]) bundle = { "target": target["name"], "decision": promotion["decision"], "winner_label": target["winner_label"], "winner_changed": promotion["winner_changed"], "next_action": promotion["next_action"], "cause_tags": promotion["causes"], "gate_status": promotion["gate_status"], "candidate_registry": registry, "review_template": { "target": target["name"], "current_description": target["winner_description"] if target["winner_label"] == "Current" else load_json(OPTIMIZATION_REPORTS[target["name"]])["current_description"], "candidate_description": target["winner_description"], "focus": promotion["causes"][:3] or ["promotion_ready"], }, "artifacts": { "skill": relative(SKILL_PATHS[target["name"]]), "optimization_report": relative(OPTIMIZATION_REPORTS[target["name"]]), "promotion_decisions": relative(ROOT / args.output_json), "candidate_registry": relative(ROOT / args.candidate_registry_json), "regression_cause_taxonomy": "references/regression-cause-taxonomy.md", "human_review_template": "references/human-review-template.md", }, } target_dir = bundle_dir / target["name"] target_dir.mkdir(parents=True, exist_ok=True) (target_dir / "bundle.json").write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (target_dir / "bundle.md").write_text(render_bundle_md(bundle), encoding="utf-8") (target_dir / "review.md").write_text( "\n".join( [ "# Human Review", "", f"- target: {bundle['review_template']['target']}", f"- current description: {bundle['review_template']['current_description']}", f"- candidate description: {bundle['review_template']['candidate_description']}", f"- suggested focus: {', '.join(bundle['review_template']['focus'])}", "", "Use the shared template in `references/human-review-template.md` to complete the final decision.", "", ] ), encoding="utf-8", ) summary = { "ok": "blocked" not in decisions, "summary": { "promote": sum(1 for d in decisions if d == "promote"), "keep_current": sum(1 for d in decisions if d == "keep_current"), "blocked": sum(1 for d in decisions if d == "blocked"), }, "targets": targets, } output_json = ROOT / args.output_json output_md = ROOT / args.output_md candidate_json = ROOT / args.candidate_registry_json candidate_md = ROOT / args.candidate_registry_md output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") output_md.write_text(render_promotion_md(targets), encoding="utf-8") candidate_json.write_text(json.dumps({"targets": targets}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") candidate_md.write_text(render_candidate_registry_md(targets), encoding="utf-8") print(json.dumps(summary, ensure_ascii=False, indent=2)) if not summary["ok"]: raise SystemExit(2) if __name__ == "__main__": main()