feat: add iteration promotion bundles
This commit is contained in:
yaojingang
@@ -0,0 +1,417 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
OPTIMIZATION_REPORTS = {
|
||||
"yao-meta-skill": ROOT / "reports" / "description_optimization.json",
|
||||
"team-frontend-review": ROOT / "examples" / "team-frontend-review" / "optimization" / "reports" / "description_optimization.json",
|
||||
"governed-incident-command": ROOT
|
||||
/ "examples"
|
||||
/ "governed-incident-command"
|
||||
/ "optimization"
|
||||
/ "reports"
|
||||
/ "description_optimization.json",
|
||||
}
|
||||
|
||||
SKILL_PATHS = {
|
||||
"yao-meta-skill": ROOT / "SKILL.md",
|
||||
"team-frontend-review": ROOT / "examples" / "team-frontend-review" / "generated-skill" / "SKILL.md",
|
||||
"governed-incident-command": ROOT / "examples" / "governed-incident-command" / "generated-skill" / "SKILL.md",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def error_total(prefix: str, target: dict) -> int:
|
||||
return int(target.get(f"{prefix}_fp", 0)) + int(target.get(f"{prefix}_fn", 0))
|
||||
|
||||
|
||||
def family_clean(summary: dict | None) -> bool:
|
||||
if not summary:
|
||||
return False
|
||||
return summary.get("clean_family_count", 0) == summary.get("family_count", 0)
|
||||
|
||||
|
||||
def relative(path: Path) -> str:
|
||||
return str(path.relative_to(ROOT))
|
||||
|
||||
|
||||
def classify_target(target: dict, route_scorecard: dict) -> dict:
|
||||
route_summary = route_scorecard["summary"]
|
||||
route_stats = route_scorecard["route_stats"].get(target["name"], {})
|
||||
winner_changed = target["winner_label"] != "Current"
|
||||
visible_ok = error_total("winner_holdout", target) <= error_total("current_holdout", target) and error_total(
|
||||
"winner_holdout", target
|
||||
) <= error_total("baseline_holdout", target)
|
||||
blind_ok = error_total("winner_blind_holdout", target) <= error_total(
|
||||
"current_blind_holdout", target
|
||||
) and error_total("winner_blind_holdout", target) <= error_total("baseline_blind_holdout", target)
|
||||
judge_ok = error_total("winner_judge_blind_holdout", target) <= error_total(
|
||||
"current_judge_blind_holdout", target
|
||||
) and error_total("winner_judge_blind_holdout", target) <= error_total("baseline_judge_blind_holdout", target)
|
||||
judge_agreement = ((target.get("judge_blind") or {}).get("winner") or {}).get("agreement_rate")
|
||||
judge_agreement_ok = judge_agreement is not None and judge_agreement >= 0.95
|
||||
adversarial_ok = error_total("winner_adversarial_holdout", target) <= error_total(
|
||||
"current_adversarial_holdout", target
|
||||
) and error_total("winner_adversarial_holdout", target) <= error_total("baseline_adversarial_holdout", target)
|
||||
holdout_calibration = (target.get("calibration") or {}).get("holdout") or {}
|
||||
adversarial_calibration = (target.get("calibration") or {}).get("adversarial_holdout") or {}
|
||||
adversarial_risk = adversarial_calibration.get("risk_band")
|
||||
adversarial_risk_ok = adversarial_risk not in {"overlap", None}
|
||||
route_clean = (
|
||||
route_summary.get("misroute_count", 0) == 0
|
||||
and route_summary.get("ambiguous_case_count", 0) == 0
|
||||
and route_stats.get("precision") == 1.0
|
||||
and route_stats.get("recall") == 1.0
|
||||
)
|
||||
family_ok = all(
|
||||
family_clean((target.get("family_health") or {}).get(gate))
|
||||
for gate in ("blind_holdout", "judge_blind_holdout", "adversarial_holdout")
|
||||
)
|
||||
|
||||
causes = []
|
||||
if not winner_changed:
|
||||
causes.append("no_candidate_outperformed_current")
|
||||
if not visible_ok:
|
||||
causes.append("visible_holdout_regression")
|
||||
if not blind_ok:
|
||||
causes.append("blind_holdout_regression")
|
||||
if error_total("winner_holdout", target) > 0:
|
||||
causes.append("current_holdout_gap_present")
|
||||
if holdout_calibration.get("risk_band") in {"overlap", "watch", "tight"}:
|
||||
causes.append("current_holdout_risk")
|
||||
if not judge_ok:
|
||||
causes.append("judge_blind_regression")
|
||||
if not judge_agreement_ok:
|
||||
causes.append("judge_blind_low_agreement")
|
||||
if not adversarial_ok:
|
||||
causes.append("adversarial_regression")
|
||||
if adversarial_risk == "overlap":
|
||||
causes.append("adversarial_overlap_risk")
|
||||
elif adversarial_risk in {"watch", "tight"}:
|
||||
causes.append("adversarial_watch_risk")
|
||||
if not family_ok:
|
||||
causes.append("family_instability")
|
||||
if route_summary.get("misroute_count", 0):
|
||||
causes.append("route_confusion")
|
||||
if route_summary.get("ambiguous_case_count", 0):
|
||||
causes.append("route_ambiguity")
|
||||
if target["winner_tokens"] > target["current_tokens"] and not winner_changed:
|
||||
causes.append("longer_without_gain")
|
||||
|
||||
severe = (
|
||||
route_summary.get("misroute_count", 0) > 0
|
||||
or route_summary.get("ambiguous_case_count", 0) > 0
|
||||
or adversarial_risk == "overlap"
|
||||
or not judge_agreement_ok
|
||||
)
|
||||
|
||||
all_gates_pass = all(
|
||||
[visible_ok, blind_ok, judge_ok, judge_agreement_ok, adversarial_ok, adversarial_risk_ok, route_clean, family_ok]
|
||||
)
|
||||
|
||||
if winner_changed and all_gates_pass:
|
||||
decision = "promote"
|
||||
elif severe:
|
||||
decision = "blocked"
|
||||
else:
|
||||
decision = "keep_current"
|
||||
|
||||
if decision == "promote" and "promotion_ready" not in causes:
|
||||
causes.append("promotion_ready")
|
||||
|
||||
next_action = {
|
||||
"promote": "Review the promoted candidate, update the skill description, and snapshot the release.",
|
||||
"keep_current": "Keep the current description and open a new candidate only when fresh route evidence appears.",
|
||||
"blocked": "Do not promote. Investigate failing gates, route collisions, or calibration risk before generating new candidates.",
|
||||
}[decision]
|
||||
|
||||
return {
|
||||
"decision": decision,
|
||||
"winner_changed": winner_changed,
|
||||
"all_gates_pass": all_gates_pass,
|
||||
"route_clean": route_clean,
|
||||
"judge_agreement_ok": judge_agreement_ok,
|
||||
"causes": causes,
|
||||
"next_action": next_action,
|
||||
"gate_status": {
|
||||
"visible_holdout_non_regression": visible_ok,
|
||||
"blind_holdout_non_regression": blind_ok,
|
||||
"judge_blind_non_regression": judge_ok,
|
||||
"judge_blind_agreement": judge_agreement_ok,
|
||||
"adversarial_non_regression": adversarial_ok,
|
||||
"adversarial_risk_ok": adversarial_risk_ok,
|
||||
"route_confusion_clean": route_clean,
|
||||
"family_stability": family_ok,
|
||||
},
|
||||
"route_metrics": {
|
||||
"precision": route_stats.get("precision"),
|
||||
"recall": route_stats.get("recall"),
|
||||
"average_margin": route_stats.get("average_margin"),
|
||||
"misroute_count": route_summary.get("misroute_count"),
|
||||
"ambiguous_case_count": route_summary.get("ambiguous_case_count"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_candidate_entries(target_name: str, target_summary: dict, decision: str) -> list[dict]:
|
||||
report = load_json(OPTIMIZATION_REPORTS[target_name])
|
||||
winner_description = report["winner"]["description"]
|
||||
winner_dev_errors = report["winner"]["dev"]["false_positives"] + report["winner"]["dev"]["false_negatives"]
|
||||
entries = [
|
||||
{
|
||||
"name": target_name,
|
||||
"role": "baseline",
|
||||
"label": "Baseline",
|
||||
"ranking_state": "reference",
|
||||
"promotion_state": "reference",
|
||||
"description": report["baseline"]["description"] if report.get("baseline") else None,
|
||||
"tokens": report["baseline"]["estimated_tokens"] if report.get("baseline") else None,
|
||||
"dev_errors": (
|
||||
report["baseline"]["dev"]["false_positives"] + report["baseline"]["dev"]["false_negatives"]
|
||||
if report.get("baseline")
|
||||
else None
|
||||
),
|
||||
"holdout_errors": (
|
||||
report["baseline"]["holdout"]["false_positives"] + report["baseline"]["holdout"]["false_negatives"]
|
||||
if report.get("baseline") and report["baseline"].get("holdout")
|
||||
else None
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
for candidate in report["candidates"]:
|
||||
selected = candidate["description"] == winner_description
|
||||
role = "current" if candidate["id"] == "current" else "candidate"
|
||||
if selected and decision == "promote":
|
||||
promotion_state = "promoted"
|
||||
elif selected and role == "current":
|
||||
promotion_state = "kept_current"
|
||||
elif selected:
|
||||
promotion_state = "blocked"
|
||||
else:
|
||||
promotion_state = "blocked"
|
||||
|
||||
reason_tags = []
|
||||
candidate_dev_errors = candidate["dev"]["false_positives"] + candidate["dev"]["false_negatives"]
|
||||
if candidate_dev_errors > winner_dev_errors:
|
||||
reason_tags.append("weaker_dev_fit")
|
||||
holdout = candidate.get("holdout") or {}
|
||||
holdout_errors = holdout.get("false_positives", 0) + holdout.get("false_negatives", 0) if holdout else None
|
||||
if holdout_errors is not None and holdout_errors > error_total("current_holdout", target_summary):
|
||||
reason_tags.append("visible_holdout_regression")
|
||||
if candidate["estimated_tokens"] > target_summary["current_tokens"] and not selected:
|
||||
reason_tags.append("longer_without_gain")
|
||||
if not reason_tags and not selected:
|
||||
reason_tags.append("not_selected_by_dev_ranking")
|
||||
|
||||
entries.append(
|
||||
{
|
||||
"name": target_name,
|
||||
"role": role,
|
||||
"label": candidate["label"],
|
||||
"ranking_state": "selected_by_dev" if selected else "not_selected",
|
||||
"promotion_state": promotion_state,
|
||||
"description": candidate["description"],
|
||||
"tokens": candidate["estimated_tokens"],
|
||||
"dev_errors": candidate_dev_errors,
|
||||
"holdout_errors": holdout_errors,
|
||||
"reason_tags": reason_tags,
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def render_candidate_registry_md(targets: list[dict]) -> str:
|
||||
lines = [
|
||||
"# Candidate Registry",
|
||||
"",
|
||||
"| Target | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | Reason Tags |",
|
||||
"| --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |",
|
||||
]
|
||||
for target in targets:
|
||||
for entry in target["registry"]:
|
||||
lines.append(
|
||||
f"| `{target['name']}` | {entry['role']} | `{entry['label']}` | {entry['ranking_state']} | {entry['promotion_state']} | {'-' if entry['tokens'] is None else entry['tokens']} | {'-' if entry['dev_errors'] is None else entry['dev_errors']} | {'-' if entry['holdout_errors'] is None else entry['holdout_errors']} | {', '.join(entry.get('reason_tags', [])) or '-'} |"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def render_promotion_md(targets: list[dict]) -> str:
|
||||
lines = [
|
||||
"# Promotion Decisions",
|
||||
"",
|
||||
"| Target | Decision | Winner | Route Clean | Judge Agreement | Adv Risk | Causes | Next Action |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for target in targets:
|
||||
decision = target["decision"]
|
||||
judge_summary = ((target.get("judge_blind") or {}).get("winner") or {}).get("agreement_rate")
|
||||
adv_risk = ((target.get("calibration") or {}).get("adversarial_holdout") or {}).get("risk_band")
|
||||
lines.append(
|
||||
f"| `{target['name']}` | `{decision}` | `{target['winner_label']}` | {target['promotion']['route_clean']} | {judge_summary if judge_summary is not None else '-'} | {adv_risk or '-'} | {', '.join(target['promotion']['causes'])} | {target['promotion']['next_action']} |"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def render_bundle_md(bundle: dict) -> str:
|
||||
review = bundle["review_template"]
|
||||
lines = [
|
||||
f"# Iteration Bundle: {bundle['target']}",
|
||||
"",
|
||||
f"- decision: `{bundle['decision']}`",
|
||||
f"- winner label: `{bundle['winner_label']}`",
|
||||
f"- winner changed: `{bundle['winner_changed']}`",
|
||||
f"- next action: {bundle['next_action']}",
|
||||
"",
|
||||
"## Cause Tags",
|
||||
"",
|
||||
]
|
||||
for cause in bundle["cause_tags"]:
|
||||
lines.append(f"- `{cause}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Gate Status",
|
||||
"",
|
||||
"| Gate | Pass |",
|
||||
"| --- | --- |",
|
||||
]
|
||||
)
|
||||
for gate, passed in bundle["gate_status"].items():
|
||||
lines.append(f"| `{gate}` | {passed} |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Candidate Registry",
|
||||
"",
|
||||
"| Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors |",
|
||||
"| --- | --- | --- | --- | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for entry in bundle["candidate_registry"]:
|
||||
lines.append(
|
||||
f"| {entry['role']} | `{entry['label']}` | {entry['ranking_state']} | {entry['promotion_state']} | {'-' if entry['tokens'] is None else entry['tokens']} | {'-' if entry['dev_errors'] is None else entry['dev_errors']} | {'-' if entry['holdout_errors'] is None else entry['holdout_errors']} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Human Review Stub",
|
||||
"",
|
||||
f"- target: {review['target']}",
|
||||
f"- current description: {review['current_description']}",
|
||||
f"- candidate description: {review['candidate_description']}",
|
||||
f"- review focus: {', '.join(review['focus'])}",
|
||||
"",
|
||||
"## Artifact Paths",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for label, path in bundle["artifacts"].items():
|
||||
lines.append(f"- {label}: `{path}`")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Apply promotion policy and build iteration bundles.")
|
||||
parser.add_argument("--optimization-suite", default="reports/description_optimization_suite.json")
|
||||
parser.add_argument("--route-scorecard", default="reports/route_scorecard.json")
|
||||
parser.add_argument("--output-json", default="reports/promotion_decisions.json")
|
||||
parser.add_argument("--output-md", default="reports/promotion_decisions.md")
|
||||
parser.add_argument("--candidate-registry-json", default="reports/candidate_registry.json")
|
||||
parser.add_argument("--candidate-registry-md", default="reports/candidate_registry.md")
|
||||
parser.add_argument("--bundle-dir", default="reports/iteration_bundles")
|
||||
args = parser.parse_args()
|
||||
|
||||
suite = load_json(ROOT / args.optimization_suite)
|
||||
route_scorecard = load_json(ROOT / args.route_scorecard)
|
||||
bundle_dir = ROOT / args.bundle_dir
|
||||
bundle_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
targets = []
|
||||
decisions = []
|
||||
for target in suite["targets"]:
|
||||
promotion = classify_target(target, route_scorecard)
|
||||
registry = build_candidate_entries(target["name"], target, promotion["decision"])
|
||||
target["promotion"] = promotion
|
||||
target["registry"] = registry
|
||||
target["decision"] = promotion["decision"]
|
||||
targets.append(target)
|
||||
decisions.append(promotion["decision"])
|
||||
|
||||
bundle = {
|
||||
"target": target["name"],
|
||||
"decision": promotion["decision"],
|
||||
"winner_label": target["winner_label"],
|
||||
"winner_changed": promotion["winner_changed"],
|
||||
"next_action": promotion["next_action"],
|
||||
"cause_tags": promotion["causes"],
|
||||
"gate_status": promotion["gate_status"],
|
||||
"candidate_registry": registry,
|
||||
"review_template": {
|
||||
"target": target["name"],
|
||||
"current_description": target["winner_description"] if target["winner_label"] == "Current" else load_json(OPTIMIZATION_REPORTS[target["name"]])["current_description"],
|
||||
"candidate_description": target["winner_description"],
|
||||
"focus": promotion["causes"][:3] or ["promotion_ready"],
|
||||
},
|
||||
"artifacts": {
|
||||
"skill": relative(SKILL_PATHS[target["name"]]),
|
||||
"optimization_report": relative(OPTIMIZATION_REPORTS[target["name"]]),
|
||||
"promotion_decisions": relative(ROOT / args.output_json),
|
||||
"candidate_registry": relative(ROOT / args.candidate_registry_json),
|
||||
"regression_cause_taxonomy": "references/regression-cause-taxonomy.md",
|
||||
"human_review_template": "references/human-review-template.md",
|
||||
},
|
||||
}
|
||||
target_dir = bundle_dir / target["name"]
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
(target_dir / "bundle.json").write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(target_dir / "bundle.md").write_text(render_bundle_md(bundle), encoding="utf-8")
|
||||
(target_dir / "review.md").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# Human Review",
|
||||
"",
|
||||
f"- target: {bundle['review_template']['target']}",
|
||||
f"- current description: {bundle['review_template']['current_description']}",
|
||||
f"- candidate description: {bundle['review_template']['candidate_description']}",
|
||||
f"- suggested focus: {', '.join(bundle['review_template']['focus'])}",
|
||||
"",
|
||||
"Use the shared template in `references/human-review-template.md` to complete the final decision.",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
summary = {
|
||||
"ok": "blocked" not in decisions,
|
||||
"summary": {
|
||||
"promote": sum(1 for d in decisions if d == "promote"),
|
||||
"keep_current": sum(1 for d in decisions if d == "keep_current"),
|
||||
"blocked": sum(1 for d in decisions if d == "blocked"),
|
||||
},
|
||||
"targets": targets,
|
||||
}
|
||||
|
||||
output_json = ROOT / args.output_json
|
||||
output_md = ROOT / args.output_md
|
||||
candidate_json = ROOT / args.candidate_registry_json
|
||||
candidate_md = ROOT / args.candidate_registry_md
|
||||
output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_promotion_md(targets), encoding="utf-8")
|
||||
candidate_json.write_text(json.dumps({"targets": targets}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
candidate_md.write_text(render_candidate_registry_md(targets), encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
if not summary["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,7 +15,9 @@ def load_snapshots(history_dir: Path) -> list[dict]:
|
||||
return [load_json(path) for path in sorted(history_dir.glob("*.json"))]
|
||||
|
||||
|
||||
def render_markdown(system_snapshots: list[dict], description_snapshots: list[dict], route_scorecard: dict) -> str:
|
||||
def render_markdown(
|
||||
system_snapshots: list[dict], description_snapshots: list[dict], route_scorecard: dict, promotion_decisions: dict
|
||||
) -> str:
|
||||
lines = [
|
||||
"# Iteration Ledger",
|
||||
"",
|
||||
@@ -53,6 +55,20 @@ def render_markdown(system_snapshots: list[dict], description_snapshots: list[di
|
||||
f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {'-' if blind_errors is None else blind_errors} | {'-' if judge_blind_errors is None else judge_blind_errors} | {judge_blind.get('agreement_rate', '-')} | {'-' if adversarial_errors is None else adversarial_errors} | {'-' if adversarial_calibration.get('score_gap') is None else adversarial_calibration.get('score_gap')} | {adversarial_calibration.get('risk_band', '-')} | {target.get('drift_note', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Current Promotion Decisions",
|
||||
"",
|
||||
"| Target | Decision | Winner | Causes | Next Action |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for target in promotion_decisions.get("targets", []):
|
||||
lines.append(
|
||||
f"| `{target['name']}` | `{target['decision']}` | `{target['winner_label']}` | {', '.join(target['promotion']['causes'])} | {target['promotion']['next_action']} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
@@ -79,17 +95,19 @@ def main() -> None:
|
||||
parser.add_argument("--history-dir", default="evals/history")
|
||||
parser.add_argument("--description-history-dir", default="evals/history/description_optimization")
|
||||
parser.add_argument("--route-scorecard", default="reports/route_scorecard.json")
|
||||
parser.add_argument("--promotion-decisions", default="reports/promotion_decisions.json")
|
||||
parser.add_argument("--output", default="reports/iteration_ledger.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
system_snapshots = load_snapshots(ROOT / args.history_dir)
|
||||
description_snapshots = load_snapshots(ROOT / args.description_history_dir)
|
||||
route_scorecard = load_json(ROOT / args.route_scorecard)
|
||||
promotion_decisions = load_json(ROOT / args.promotion_decisions)
|
||||
|
||||
output = ROOT / args.output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
render_markdown(system_snapshots, description_snapshots, route_scorecard),
|
||||
render_markdown(system_snapshots, description_snapshots, route_scorecard, promotion_decisions),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user