wehub-resource-sync
78 lines
3.3 KiB
Python
78 lines
3.3 KiB
Python
"""Baseline diff (regression detector) + baseline construction.
|
|
|
|
An absolute ``floor`` is the hard fail; on top, a per-metric drop beyond
|
|
``judge_tolerance`` flags drift even while still above the floor.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
|
|
def diff_against_baseline(
|
|
current: dict[str, dict[str, Any]], baseline: dict[str, Any]
|
|
) -> dict[str, Any]:
|
|
"""Compare this run's per-variation metrics against the committed baseline."""
|
|
floor = baseline.get("floor", {})
|
|
tol = baseline.get("judge_tolerance", 1)
|
|
base_vars = baseline.get("variations", {})
|
|
regressions: list[dict[str, Any]] = []
|
|
|
|
for jd_key, cur in current.items():
|
|
base = base_vars.get(jd_key, {})
|
|
cov = cur.get("jd_keyword_coverage")
|
|
judge = cur.get("judge_score")
|
|
non_blank = cur.get("non_blank")
|
|
|
|
if cov is not None and cov < floor.get("min_keyword_coverage", 0.0):
|
|
regressions.append({"jd_key": jd_key, "kind": "keyword_floor", "value": cov})
|
|
if judge is None and base.get("judge_score") is not None:
|
|
# The judge produced a score for this variation at baseline but nothing
|
|
# now (e.g. it errored) — worse than any low score, so flag it.
|
|
regressions.append({
|
|
"jd_key": jd_key,
|
|
"kind": "judge_missing",
|
|
"baseline_value": base.get("judge_score"),
|
|
})
|
|
elif judge is not None and judge < floor.get("min_judge_score", 0):
|
|
regressions.append({"jd_key": jd_key, "kind": "judge_floor", "value": judge})
|
|
if non_blank is False:
|
|
regressions.append({"jd_key": jd_key, "kind": "blank_render", "value": False})
|
|
base_judge = base.get("judge_score")
|
|
if (
|
|
isinstance(judge, int) and not isinstance(judge, bool)
|
|
and isinstance(base_judge, int) and not isinstance(base_judge, bool)
|
|
and (base_judge - judge) > tol
|
|
):
|
|
regressions.append(
|
|
{"jd_key": jd_key, "kind": "judge_drop", "from": base_judge, "to": judge}
|
|
)
|
|
|
|
for jd_key in base_vars:
|
|
if jd_key not in current:
|
|
regressions.append({"jd_key": jd_key, "kind": "missing_variation"})
|
|
|
|
return {"regressed": bool(regressions), "regressions": regressions}
|
|
|
|
|
|
def summary_to_baseline(variations: list[dict[str, Any]]) -> dict[str, Any]:
|
|
"""Build a baseline ``variations`` block from a run's variation results."""
|
|
out: dict[str, Any] = {
|
|
"variations": {},
|
|
# Floors are the absolute "this is broken" bar; per-variation drift
|
|
# (judge_tolerance) catches regressions above the floor. The fixture set
|
|
# deliberately includes JDs far from the master (frontend/ML/PM) whose
|
|
# truthful tailoring legitimately scores ~2 — so the judge floor sits at
|
|
# 2, not 3, to avoid false-positives on those honest-but-weak variations.
|
|
"floor": {"min_judge_score": 2, "min_keyword_coverage": 0.5},
|
|
"judge_tolerance": 1,
|
|
}
|
|
for v in variations:
|
|
scores = v.get("scores", {})
|
|
out["variations"][v["jd_key"]] = {
|
|
"jd_keyword_coverage": scores.get("jd_keyword_coverage"),
|
|
"judge_score": (v.get("judge") or {}).get("score"),
|
|
"non_blank": (v.get("render") or {}).get("non_blank"),
|
|
}
|
|
return out
|