yaojingang
67 lines
1.8 KiB
Python
67 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FAILURES_DIR = ROOT / "failures"
|
|
TRIGGER_EVAL = ROOT / "scripts" / "trigger_eval.py"
|
|
DESCRIPTION = ROOT / "evals" / "improved_description.txt"
|
|
BASELINE = ROOT / "evals" / "baseline_description.txt"
|
|
SEMANTIC_CONFIG = ROOT / "evals" / "semantic_config.json"
|
|
|
|
|
|
def run_case(case_file: Path) -> dict:
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(TRIGGER_EVAL),
|
|
"--description-file",
|
|
str(DESCRIPTION),
|
|
"--baseline-description-file",
|
|
str(BASELINE),
|
|
"--cases",
|
|
str(case_file),
|
|
"--semantic-config",
|
|
str(SEMANTIC_CONFIG),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
payload = json.loads(proc.stdout)
|
|
payload["returncode"] = proc.returncode
|
|
payload["case_file"] = str(case_file.relative_to(ROOT))
|
|
return payload
|
|
|
|
|
|
def main() -> None:
|
|
case_files = sorted(FAILURES_DIR.glob("*/cases.json"))
|
|
reports = [run_case(path) for path in case_files]
|
|
failures = []
|
|
for report in reports:
|
|
if report["returncode"] != 0:
|
|
failures.append(
|
|
{
|
|
"case_file": report["case_file"],
|
|
"false_positives": report["false_positives"],
|
|
"false_negatives": report["false_negatives"],
|
|
"misfires": report["misfires"],
|
|
}
|
|
)
|
|
|
|
output = {
|
|
"ok": not failures,
|
|
"case_count": len(reports),
|
|
"families": [report["case_file"] for report in reports],
|
|
"failures": failures,
|
|
}
|
|
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
if failures:
|
|
raise SystemExit(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|