feat: expand family suite and publish eval dashboard

scripts/render_eval_dashboard.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+BEGIN_MARKER = "<!-- BEGIN:EVAL_RESULTS -->"
+END_MARKER = "<!-- END:EVAL_RESULTS -->"
+
+
+def run_eval_suite(args: argparse.Namespace) -> dict:
+    proc = subprocess.run(
+        [
+            sys.executable,
+            "scripts/run_eval_suite.py",
+            "--eval-dir",
+            args.eval_dir,
+            "--description-file",
+            args.description_file,
+            "--baseline-description-file",
+            args.baseline_description_file,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(proc.stdout)
+
+
+def render_readme_panel(report: dict) -> str:
+    summary = report["summary"]
+    suites = report["suites"]
+    families = report["family_summary"]
+
+    lines = [
+        BEGIN_MARKER,
+        f"- regression corpus: `{summary['total_cases']}` prompts across `{summary['family_count']}` families",
+        f"- aggregate result: `{summary['false_positives']}` false positives, `{summary['false_negatives']}` false negatives, average precision `{summary['average_precision']}`, average recall `{summary['average_recall']}`",
+        "- suite status:",
+        "",
+        "| Suite | Cases | FP | FN | Precision | Recall |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for suite_name in ("train", "dev", "holdout"):
+        suite = suites[suite_name]
+        cases = sum(bucket["total"] for bucket in suite["bucket_stats"].values())
+        lines.append(
+            f"| {suite_name} | {cases} | {suite['false_positives']} | {suite['false_negatives']} | {suite['precision']} | {suite['recall']} |"
+        )
+
+    lines.extend(
+        [
+            "",
+            "| Family | Cases | Pass Rate |",
+            "| --- | ---: | ---: |",
+        ]
+    )
+    for family, stats in sorted(families.items()):
+        lines.append(f"| `{family}` | {stats['total']} | {stats['pass_rate']} |")
+
+    lines.extend(
+        [
+            "",
+            "Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)",
+            END_MARKER,
+        ]
+    )
+    return "\n".join(lines)
+
+
+def render_family_summary(report: dict) -> str:
+    summary = report["summary"]
+    lines = [
+        "# Eval Family Summary",
+        "",
+        f"- total cases: `{summary['total_cases']}`",
+        f"- families: `{summary['family_count']}`",
+        f"- false positives: `{summary['false_positives']}`",
+        f"- false negatives: `{summary['false_negatives']}`",
+        f"- average precision: `{summary['average_precision']}`",
+        f"- average recall: `{summary['average_recall']}`",
+        "",
+        "## Family Results",
+        "",
+        "| Family | Cases | Passed | FP | FN | Pass Rate |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for family, stats in sorted(report["family_summary"].items()):
+        lines.append(
+            f"| `{family}` | {stats['total']} | {stats['passed']} | {stats['false_positives']} | {stats['false_negatives']} | {stats['pass_rate']} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def update_readme(readme_path: Path, panel: str) -> None:
+    text = readme_path.read_text(encoding="utf-8")
+    if BEGIN_MARKER not in text or END_MARKER not in text:
+        raise SystemExit(f"README markers not found in {readme_path}")
+    start = text.index(BEGIN_MARKER)
+    end = text.index(END_MARKER) + len(END_MARKER)
+    updated = text[:start] + panel + text[end:]
+    readme_path.write_text(updated, encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Render README-visible eval results and full family report.")
+    parser.add_argument("--eval-dir", default="evals")
+    parser.add_argument("--description-file", default="evals/improved_description.txt")
+    parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt")
+    parser.add_argument("--output-dir", default="reports")
+    parser.add_argument("--readme", default="README.md")
+    args = parser.parse_args()
+
+    report = run_eval_suite(args)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    (output_dir / "eval_suite.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    (output_dir / "family_summary.md").write_text(render_family_summary(report), encoding="utf-8")
+    update_readme(Path(args.readme), render_readme_panel(report))
+
+
+if __name__ == "__main__":
+    main()

scripts/run_eval_suite.py

@@ -33,6 +33,7 @@ def main() -> None:
     parser.add_argument("--eval-dir", default="evals", help="Root eval directory")
     parser.add_argument("--description-file", default="evals/improved_description.txt")
     parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt")
+    parser.add_argument("--output-file", help="Optional path to write the combined JSON report")
     args = parser.parse_args()
 
     root = Path(args.eval_dir).resolve()
@@ -65,15 +66,21 @@ def main() -> None:
     for family, stats in family_summary.items():
         stats["pass_rate"] = round(stats["passed"] / stats["total"], 3) if stats["total"] else None
 
+    total_cases = sum(stats["total"] for stats in family_summary.values())
     summary = {
         "suite_count": len(suites),
+        "total_cases": total_cases,
+        "family_count": len(family_summary),
         "false_positives": aggregate["false_positives"],
         "false_negatives": aggregate["false_negatives"],
         "average_precision": round(sum(aggregate["precision"]) / len(aggregate["precision"]), 3) if aggregate["precision"] else None,
         "average_recall": round(sum(aggregate["recall"]) / len(aggregate["recall"]), 3) if aggregate["recall"] else None,
     }
     output = {"summary": summary, "family_summary": family_summary, "suites": suites}
-    print(json.dumps(output, ensure_ascii=False, indent=2))
+    rendered = json.dumps(output, ensure_ascii=False, indent=2)
+    if args.output_file:
+        Path(args.output_file).write_text(rendered + "\n", encoding="utf-8")
+    print(rendered)
     if summary["false_positives"] > 0 or summary["false_negatives"] > 0:
         raise SystemExit(2)