yaojingang
127 lines
4.5 KiB
Python
127 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
BEGIN_MARKER = "<!-- BEGIN:EVAL_RESULTS -->"
|
|
END_MARKER = "<!-- END:EVAL_RESULTS -->"
|
|
|
|
|
|
def run_eval_suite(args: argparse.Namespace) -> dict:
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"scripts/run_eval_suite.py",
|
|
"--eval-dir",
|
|
args.eval_dir,
|
|
"--description-file",
|
|
args.description_file,
|
|
"--baseline-description-file",
|
|
args.baseline_description_file,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return json.loads(proc.stdout)
|
|
|
|
|
|
def render_readme_panel(report: dict) -> str:
|
|
summary = report["summary"]
|
|
suites = report["suites"]
|
|
families = report["family_summary"]
|
|
|
|
lines = [
|
|
BEGIN_MARKER,
|
|
f"- regression corpus: `{summary['total_cases']}` prompts across `{summary['family_count']}` families",
|
|
f"- aggregate result: `{summary['false_positives']}` false positives, `{summary['false_negatives']}` false negatives, average precision `{summary['average_precision']}`, average recall `{summary['average_recall']}`",
|
|
"- suite status:",
|
|
"",
|
|
"| Suite | Cases | FP | FN | Precision | Recall |",
|
|
"| --- | ---: | ---: | ---: | ---: | ---: |",
|
|
]
|
|
for suite_name in ("train", "dev", "holdout"):
|
|
suite = suites[suite_name]
|
|
cases = sum(bucket["total"] for bucket in suite["bucket_stats"].values())
|
|
lines.append(
|
|
f"| {suite_name} | {cases} | {suite['false_positives']} | {suite['false_negatives']} | {suite['precision']} | {suite['recall']} |"
|
|
)
|
|
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"| Family | Cases | Pass Rate |",
|
|
"| --- | ---: | ---: |",
|
|
]
|
|
)
|
|
for family, stats in sorted(families.items()):
|
|
lines.append(f"| `{family}` | {stats['total']} | {stats['pass_rate']} |")
|
|
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)",
|
|
END_MARKER,
|
|
]
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_family_summary(report: dict) -> str:
|
|
summary = report["summary"]
|
|
lines = [
|
|
"# Eval Family Summary",
|
|
"",
|
|
f"- total cases: `{summary['total_cases']}`",
|
|
f"- families: `{summary['family_count']}`",
|
|
f"- false positives: `{summary['false_positives']}`",
|
|
f"- false negatives: `{summary['false_negatives']}`",
|
|
f"- average precision: `{summary['average_precision']}`",
|
|
f"- average recall: `{summary['average_recall']}`",
|
|
"",
|
|
"## Family Results",
|
|
"",
|
|
"| Family | Cases | Passed | FP | FN | Pass Rate |",
|
|
"| --- | ---: | ---: | ---: | ---: | ---: |",
|
|
]
|
|
for family, stats in sorted(report["family_summary"].items()):
|
|
lines.append(
|
|
f"| `{family}` | {stats['total']} | {stats['passed']} | {stats['false_positives']} | {stats['false_negatives']} | {stats['pass_rate']} |"
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def update_readme(readme_path: Path, panel: str) -> None:
|
|
text = readme_path.read_text(encoding="utf-8")
|
|
if BEGIN_MARKER not in text or END_MARKER not in text:
|
|
raise SystemExit(f"README markers not found in {readme_path}")
|
|
start = text.index(BEGIN_MARKER)
|
|
end = text.index(END_MARKER) + len(END_MARKER)
|
|
updated = text[:start] + panel + text[end:]
|
|
readme_path.write_text(updated, encoding="utf-8")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Render README-visible eval results and full family report.")
|
|
parser.add_argument("--eval-dir", default="evals")
|
|
parser.add_argument("--description-file", default="evals/improved_description.txt")
|
|
parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt")
|
|
parser.add_argument("--output-dir", default="reports")
|
|
parser.add_argument("--readme", default="README.md")
|
|
args = parser.parse_args()
|
|
|
|
report = run_eval_suite(args)
|
|
output_dir = Path(args.output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
(output_dir / "eval_suite.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
(output_dir / "family_summary.md").write_text(render_family_summary(report), encoding="utf-8")
|
|
update_readme(Path(args.readme), render_readme_panel(report))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|