feat: expand family suite and publish eval dashboard

2026-03-31 21:42:39 +08:00
parent 51a479d12f
commit bcaddedb58
13 changed files with 5599 additions and 30 deletions
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+BEGIN_MARKER = "<!-- BEGIN:EVAL_RESULTS -->"
+END_MARKER = "<!-- END:EVAL_RESULTS -->"
+
+
+def run_eval_suite(args: argparse.Namespace) -> dict:
+    proc = subprocess.run(
+        [
+            sys.executable,
+            "scripts/run_eval_suite.py",
+            "--eval-dir",
+            args.eval_dir,
+            "--description-file",
+            args.description_file,
+            "--baseline-description-file",
+            args.baseline_description_file,
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(proc.stdout)
+
+
+def render_readme_panel(report: dict) -> str:
+    summary = report["summary"]
+    suites = report["suites"]
+    families = report["family_summary"]
+
+    lines = [
+        BEGIN_MARKER,
+        f"- regression corpus: `{summary['total_cases']}` prompts across `{summary['family_count']}` families",
+        f"- aggregate result: `{summary['false_positives']}` false positives, `{summary['false_negatives']}` false negatives, average precision `{summary['average_precision']}`, average recall `{summary['average_recall']}`",
+        "- suite status:",
+        "",
+        "| Suite | Cases | FP | FN | Precision | Recall |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for suite_name in ("train", "dev", "holdout"):
+        suite = suites[suite_name]
+        cases = sum(bucket["total"] for bucket in suite["bucket_stats"].values())
+        lines.append(
+            f"| {suite_name} | {cases} | {suite['false_positives']} | {suite['false_negatives']} | {suite['precision']} | {suite['recall']} |"
+        )
+
+    lines.extend(
+        [
+            "",
+            "| Family | Cases | Pass Rate |",
+            "| --- | ---: | ---: |",
+        ]
+    )
+    for family, stats in sorted(families.items()):
+        lines.append(f"| `{family}` | {stats['total']} | {stats['pass_rate']} |")
+
+    lines.extend(
+        [
+            "",
+            "Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)",
+            END_MARKER,
+        ]
+    )
+    return "\n".join(lines)
+
+
+def render_family_summary(report: dict) -> str:
+    summary = report["summary"]
+    lines = [
+        "# Eval Family Summary",
+        "",
+        f"- total cases: `{summary['total_cases']}`",
+        f"- families: `{summary['family_count']}`",
+        f"- false positives: `{summary['false_positives']}`",
+        f"- false negatives: `{summary['false_negatives']}`",
+        f"- average precision: `{summary['average_precision']}`",
+        f"- average recall: `{summary['average_recall']}`",
+        "",
+        "## Family Results",
+        "",
+        "| Family | Cases | Passed | FP | FN | Pass Rate |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for family, stats in sorted(report["family_summary"].items()):
+        lines.append(
+            f"| `{family}` | {stats['total']} | {stats['passed']} | {stats['false_positives']} | {stats['false_negatives']} | {stats['pass_rate']} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def update_readme(readme_path: Path, panel: str) -> None:
+    text = readme_path.read_text(encoding="utf-8")
+    if BEGIN_MARKER not in text or END_MARKER not in text:
+        raise SystemExit(f"README markers not found in {readme_path}")
+    start = text.index(BEGIN_MARKER)
+    end = text.index(END_MARKER) + len(END_MARKER)
+    updated = text[:start] + panel + text[end:]
+    readme_path.write_text(updated, encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Render README-visible eval results and full family report.")
+    parser.add_argument("--eval-dir", default="evals")
+    parser.add_argument("--description-file", default="evals/improved_description.txt")
+    parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt")
+    parser.add_argument("--output-dir", default="reports")
+    parser.add_argument("--readme", default="README.md")
+    args = parser.parse_args()
+
+    report = run_eval_suite(args)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    (output_dir / "eval_suite.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    (output_dir / "family_summary.md").write_text(render_family_summary(report), encoding="utf-8")
+    update_readme(Path(args.readme), render_readme_panel(report))
+
+
+if __name__ == "__main__":
+    main()