chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Register and query quarantined (contaminated) runs in the experiment ledger.
|
||||
|
||||
A contamination class tags artifact batches whose results are confounded by an
|
||||
infra defect (e.g. a provider-compaction marker leak). Registration is
|
||||
idempotent: names merge into ``contaminations.json`` at the ledger root,
|
||||
matching ledger run dirs get a ``contamination.json`` stamp, and an event is
|
||||
appended to ``experiments.jsonl``. Baseline tooling and ``exp_status`` read the
|
||||
registry to warn when a quarantined artifact backs a baseline.
|
||||
|
||||
Usage:
|
||||
exp_quarantine.py register --contamination-class NAME --names-file F \
|
||||
--description TEXT [--evidence PATH] [--boundary-commit C ...]
|
||||
exp_quarantine.py check ARTIFACT [ARTIFACT ...]
|
||||
exp_quarantine.py list
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
artifact_basename,
|
||||
atomic_write_json,
|
||||
contamination_class_for,
|
||||
contaminations_path,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
load_contaminations,
|
||||
now_iso,
|
||||
read_json,
|
||||
)
|
||||
|
||||
CLASS_NAME_RE_HELP = "lowercase snake_case, e.g. tool_result_compaction_defect"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
register = sub.add_parser("register", help="register/merge a contamination class")
|
||||
register.add_argument(
|
||||
"--contamination-class",
|
||||
required=True,
|
||||
help=f"class name ({CLASS_NAME_RE_HELP})",
|
||||
)
|
||||
register.add_argument(
|
||||
"--names-file",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="JSON file containing a list of artifact batch names",
|
||||
)
|
||||
register.add_argument("--description", required=True)
|
||||
register.add_argument(
|
||||
"--evidence",
|
||||
default="",
|
||||
help="path or URL of the audit/report that established the contamination",
|
||||
)
|
||||
register.add_argument(
|
||||
"--boundary-commit",
|
||||
action="append",
|
||||
default=[],
|
||||
help="fix-boundary commit (repeatable); runs at or before these are affected",
|
||||
)
|
||||
|
||||
check = sub.add_parser("check", help="check artifact names/paths against the registry")
|
||||
check.add_argument("artifacts", nargs="+")
|
||||
|
||||
sub.add_parser("list", help="summarize registered contamination classes")
|
||||
return parser
|
||||
|
||||
|
||||
def _load_names(path: Path) -> list[str]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise LedgerError(f"cannot read names file {path}: {exc}") from exc
|
||||
if not isinstance(data, list) or not all(isinstance(item, str) for item in data):
|
||||
raise LedgerError(f"names file must be a JSON list of strings: {path}")
|
||||
# Normalize before filtering: a raw value like "/" or "///" is non-empty
|
||||
# pre-normalization but collapses to "" via artifact_basename, and an
|
||||
# empty name would substring-match (or exact-match) every run.
|
||||
names = sorted(
|
||||
{
|
||||
normalized
|
||||
for item in data
|
||||
if item.strip()
|
||||
for normalized in [artifact_basename(item)]
|
||||
if normalized
|
||||
}
|
||||
)
|
||||
if not names:
|
||||
raise LedgerError(f"names file is empty: {path}")
|
||||
return names
|
||||
|
||||
|
||||
def _candidate_names_in_payload(payload: Any) -> set[str]:
|
||||
"""Recursively collect basename-normalized string values from a JSON payload."""
|
||||
names: set[str] = set()
|
||||
if isinstance(payload, str):
|
||||
normalized = artifact_basename(payload)
|
||||
if normalized:
|
||||
names.add(normalized)
|
||||
elif isinstance(payload, dict):
|
||||
for value in payload.values():
|
||||
names.update(_candidate_names_in_payload(value))
|
||||
elif isinstance(payload, list):
|
||||
for item in payload:
|
||||
names.update(_candidate_names_in_payload(item))
|
||||
return names
|
||||
|
||||
|
||||
def _stamp_run_dirs(root: Path, class_name: str, names: list[str]) -> list[str]:
|
||||
"""Stamp ledger run dirs whose artifacts reference a quarantined batch name.
|
||||
|
||||
Matching is by exact basename, consistent with ``contamination_class_for``.
|
||||
A prior substring-based check over the raw JSON text could both false-
|
||||
positive (a quarantined name that is a substring of an unrelated, longer
|
||||
name) and, worse, treat an empty name as a substring of everything.
|
||||
"""
|
||||
stamped: list[str] = []
|
||||
runs_dir = root / "runs"
|
||||
if not runs_dir.is_dir():
|
||||
return stamped
|
||||
name_set = set(names)
|
||||
for run_dir in sorted(runs_dir.iterdir()):
|
||||
if not run_dir.is_dir():
|
||||
continue
|
||||
matched: set[str] = set()
|
||||
for source in ("artifacts.json", "manifest.json"):
|
||||
payload = read_json(run_dir / source)
|
||||
if not payload:
|
||||
continue
|
||||
matched.update(name_set & _candidate_names_in_payload(payload))
|
||||
if not matched:
|
||||
continue
|
||||
stamp_path = run_dir / "contamination.json"
|
||||
existing = read_json(stamp_path)
|
||||
classes = existing.get("classes") if isinstance(existing.get("classes"), dict) else {}
|
||||
previous = classes.get(class_name, {})
|
||||
previous_names = (
|
||||
set(previous.get("matched_artifact_names", []))
|
||||
if isinstance(previous, dict)
|
||||
else set()
|
||||
)
|
||||
classes[class_name] = {
|
||||
"matched_artifact_names": sorted(previous_names | matched),
|
||||
"stamped_at": (
|
||||
previous.get("stamped_at")
|
||||
if isinstance(previous, dict) and previous.get("stamped_at")
|
||||
else now_iso()
|
||||
),
|
||||
}
|
||||
atomic_write_json(stamp_path, {"classes": classes})
|
||||
stamped.append(run_dir.name)
|
||||
return stamped
|
||||
|
||||
|
||||
def cmd_register(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
class_name = str(args.contamination_class).strip()
|
||||
if not class_name:
|
||||
raise LedgerError("--contamination-class must be non-empty")
|
||||
names = _load_names(args.names_file)
|
||||
with ledger_lock(root):
|
||||
data = load_contaminations(root)
|
||||
classes = data["classes"]
|
||||
entry = classes.get(class_name)
|
||||
if not isinstance(entry, dict):
|
||||
entry = {"registered_at": now_iso()}
|
||||
existing_names = set(entry.get("artifact_names", []))
|
||||
merged = sorted(existing_names | set(names))
|
||||
new_names = sorted(set(names) - existing_names)
|
||||
entry.update(
|
||||
{
|
||||
"description": str(args.description),
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"artifact_names": merged,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
)
|
||||
classes[class_name] = entry
|
||||
data["updated_at"] = now_iso()
|
||||
atomic_write_json(contaminations_path(root), data)
|
||||
stamped = _stamp_run_dirs(root, class_name, merged)
|
||||
append_jsonl(
|
||||
root / "experiments.jsonl",
|
||||
{
|
||||
"event": "contamination_registered",
|
||||
"contamination_class": class_name,
|
||||
"artifact_names_total": len(merged),
|
||||
"artifact_names_new": len(new_names),
|
||||
"stamped_run_dirs": stamped,
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"time": now_iso(),
|
||||
},
|
||||
)
|
||||
print(
|
||||
f"registered {class_name}: {len(merged)} artifact names "
|
||||
f"({len(new_names)} new), stamped {len(stamped)} ledger run dirs"
|
||||
)
|
||||
for run_name in stamped:
|
||||
print(f" stamped: runs/{run_name}/contamination.json")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
contaminations = load_contaminations(root)
|
||||
dirty = 0
|
||||
for artifact in args.artifacts:
|
||||
contamination_class = contamination_class_for(root, artifact, contaminations)
|
||||
if contamination_class:
|
||||
dirty += 1
|
||||
print(f"QUARANTINED\t{artifact_basename(artifact)}\t{contamination_class}")
|
||||
else:
|
||||
print(f"clean\t{artifact_basename(artifact)}")
|
||||
return 1 if dirty else 0
|
||||
|
||||
|
||||
def cmd_list(_: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
classes = load_contaminations(root).get("classes", {})
|
||||
if not classes:
|
||||
print("no contamination classes registered")
|
||||
return 0
|
||||
for name, payload in sorted(classes.items()):
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
count = len(payload.get("artifact_names", []))
|
||||
boundary = ",".join(payload.get("boundary_commits", [])) or "-"
|
||||
print(f"{name}\truns={count}\tboundary={boundary}")
|
||||
description = payload.get("description")
|
||||
if description:
|
||||
print(f" {description}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "register":
|
||||
return cmd_register(args)
|
||||
if args.command == "check":
|
||||
return cmd_check(args)
|
||||
return cmd_list(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user