chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,361 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check Qwen/DashScope provider payload parity invariants from raw traces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DASHSCOPE_CACHE_MARKER_LIMIT = 4
|
||||
|
||||
|
||||
def _iter_json_values(path: Path) -> Iterator[Any]:
|
||||
if path.is_dir():
|
||||
for child in sorted(path.rglob("*")):
|
||||
if child.is_file() and child.suffix.lower() in {".json", ".jsonl"}:
|
||||
yield from _iter_json_values(child)
|
||||
return
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return
|
||||
if path.suffix.lower() == ".jsonl":
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
yield {"__source_path": str(path), "__value": value}
|
||||
return
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return
|
||||
yield {"__source_path": str(path), "__value": value}
|
||||
|
||||
|
||||
def _walk_dicts(value: Any) -> Iterator[dict[str, Any]]:
|
||||
if isinstance(value, dict):
|
||||
yield value
|
||||
for child in value.values():
|
||||
yield from _walk_dicts(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
yield from _walk_dicts(child)
|
||||
|
||||
|
||||
def _payloads_from_value(source_path: str, value: Any) -> Iterator[dict[str, Any]]:
|
||||
seen_payload_ids: set[int] = set()
|
||||
for obj in _walk_dicts(value):
|
||||
payload = obj.get("payload")
|
||||
if isinstance(payload, dict) and payload.get("model") and isinstance(
|
||||
payload.get("messages"),
|
||||
list,
|
||||
):
|
||||
seen_payload_ids.add(id(payload))
|
||||
yield {
|
||||
"source_path": source_path,
|
||||
"instance_id": _instance_id_from_path(source_path),
|
||||
"payload": payload,
|
||||
}
|
||||
elif (
|
||||
id(obj) not in seen_payload_ids
|
||||
and obj.get("model")
|
||||
and isinstance(obj.get("messages"), list)
|
||||
):
|
||||
yield {
|
||||
"source_path": source_path,
|
||||
"instance_id": _instance_id_from_path(source_path),
|
||||
"payload": obj,
|
||||
}
|
||||
|
||||
|
||||
def _instance_id_from_path(source_path: str) -> str:
|
||||
path = Path(source_path)
|
||||
if path.name in {"llm_calls.jsonl", "provider_trace.jsonl", "request_proof.jsonl"}:
|
||||
return path.parent.name
|
||||
if path.parent.name:
|
||||
return path.parent.name
|
||||
return ""
|
||||
|
||||
|
||||
def _extra_body(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
extra = payload.get("extra_body")
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
|
||||
|
||||
def _thinking_enabled(payload: dict[str, Any]) -> bool:
|
||||
extra = _extra_body(payload)
|
||||
return bool(
|
||||
extra.get("enable_thinking")
|
||||
or payload.get("enable_thinking")
|
||||
or payload.get("thinking")
|
||||
or payload.get("reasoning")
|
||||
)
|
||||
|
||||
|
||||
def _message_reasoning_replayed(payload: dict[str, Any]) -> bool:
|
||||
for message in payload.get("messages") or []:
|
||||
if isinstance(message, dict) and message.get("role") == "assistant":
|
||||
reasoning = message.get("reasoning_content")
|
||||
if isinstance(reasoning, str) and reasoning.strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _tool_call_pairing_ok(payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
pending: list[str] = []
|
||||
for message in payload.get("messages") or []:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
if message.get("role") == "assistant":
|
||||
for call in message.get("tool_calls") or []:
|
||||
if isinstance(call, dict) and call.get("id"):
|
||||
pending.append(str(call["id"]))
|
||||
elif message.get("role") == "tool":
|
||||
tool_call_id = message.get("tool_call_id")
|
||||
if tool_call_id in pending:
|
||||
pending.remove(tool_call_id)
|
||||
if pending:
|
||||
return False, f"unpaired assistant tool_call ids: {','.join(pending[:5])}"
|
||||
return True, "assistant tool calls and tool results are paired"
|
||||
|
||||
|
||||
_BOOLEAN_SCHEMA_KEYWORD_ALLOWLIST = {
|
||||
"additionalProperties",
|
||||
"deprecated",
|
||||
"nullable",
|
||||
"strict",
|
||||
"uniqueItems",
|
||||
}
|
||||
|
||||
|
||||
def _boolean_schema_paths(value: Any, prefix: str = "$", *, key: str | None = None) -> list[str]:
|
||||
if isinstance(value, bool):
|
||||
if key in _BOOLEAN_SCHEMA_KEYWORD_ALLOWLIST:
|
||||
return []
|
||||
return [prefix]
|
||||
if isinstance(value, dict):
|
||||
paths: list[str] = []
|
||||
for key, child in value.items():
|
||||
paths.extend(_boolean_schema_paths(child, f"{prefix}.{key}", key=key))
|
||||
return paths
|
||||
if isinstance(value, list):
|
||||
paths: list[str] = []
|
||||
for index, child in enumerate(value):
|
||||
paths.extend(_boolean_schema_paths(child, f"{prefix}[{index}]"))
|
||||
return paths
|
||||
return []
|
||||
|
||||
|
||||
def _cache_marker_count(payload: dict[str, Any]) -> int:
|
||||
count = 0
|
||||
for obj in _walk_dicts(payload.get("messages") or []):
|
||||
if "cache_control" in obj:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _row(
|
||||
*,
|
||||
source_path: str,
|
||||
instance_id: str,
|
||||
model: str,
|
||||
check: str,
|
||||
status: str,
|
||||
detail: str,
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"source_path": source_path,
|
||||
"instance_id": instance_id,
|
||||
"model": model,
|
||||
"check": check,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
|
||||
def _check_payload(
|
||||
source_path: str,
|
||||
instance_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> list[dict[str, str]]:
|
||||
model = str(payload.get("model") or "")
|
||||
model_lower = model.lower()
|
||||
qwen_flash = "qwen3.6-flash" in model_lower
|
||||
thinking = _thinking_enabled(payload)
|
||||
extra = _extra_body(payload)
|
||||
rows: list[dict[str, str]] = []
|
||||
|
||||
def add(check: str, status: str, detail: str) -> None:
|
||||
rows.append(
|
||||
_row(
|
||||
source_path=source_path,
|
||||
instance_id=instance_id,
|
||||
model=model,
|
||||
check=check,
|
||||
status=status,
|
||||
detail=detail,
|
||||
)
|
||||
)
|
||||
|
||||
if thinking:
|
||||
add(
|
||||
"dashscope_enable_thinking",
|
||||
"pass"
|
||||
if extra.get("enable_thinking") is True or payload.get("enable_thinking") is True
|
||||
else "fail",
|
||||
"enable_thinking is true"
|
||||
if extra.get("enable_thinking") is True or payload.get("enable_thinking") is True
|
||||
else "thinking appears enabled but enable_thinking is not true",
|
||||
)
|
||||
add(
|
||||
"dashscope_max_completion_tokens",
|
||||
"pass" if "max_completion_tokens" in payload else "fail",
|
||||
"max_completion_tokens present"
|
||||
if "max_completion_tokens" in payload
|
||||
else "DashScope reasoning payload should use max_completion_tokens",
|
||||
)
|
||||
forced_tool_choice = payload.get("tool_choice")
|
||||
forced_tool_choice_allowed = forced_tool_choice is None or forced_tool_choice == "auto"
|
||||
add(
|
||||
"dashscope_thinking_no_forced_tool_choice",
|
||||
"pass" if forced_tool_choice_allowed else "fail",
|
||||
"no forced tool_choice during thinking"
|
||||
if forced_tool_choice_allowed
|
||||
else "forced tool_choice present during thinking",
|
||||
)
|
||||
else:
|
||||
add("dashscope_enable_thinking", "skip", "thinking not detected")
|
||||
add("dashscope_max_completion_tokens", "skip", "thinking not detected")
|
||||
add("dashscope_thinking_no_forced_tool_choice", "skip", "thinking not detected")
|
||||
|
||||
if qwen_flash:
|
||||
add(
|
||||
"qwen_flash_no_reasoning_replay",
|
||||
"fail" if _message_reasoning_replayed(payload) else "pass",
|
||||
"historical assistant reasoning_content replayed"
|
||||
if _message_reasoning_replayed(payload)
|
||||
else "no historical reasoning_content replay",
|
||||
)
|
||||
preserve_thinking = bool(extra.get("preserve_thinking") or payload.get("preserve_thinking"))
|
||||
add(
|
||||
"qwen_flash_no_preserve_thinking",
|
||||
"fail" if preserve_thinking else "pass",
|
||||
"preserve_thinking present for qwen3.6-flash"
|
||||
if preserve_thinking
|
||||
else "preserve_thinking absent for qwen3.6-flash",
|
||||
)
|
||||
else:
|
||||
add("qwen_flash_no_reasoning_replay", "skip", "not qwen3.6-flash")
|
||||
add("qwen_flash_no_preserve_thinking", "skip", "not qwen3.6-flash")
|
||||
|
||||
stream_options = payload.get("stream_options")
|
||||
if payload.get("stream") is False:
|
||||
add("stream_include_usage", "skip", "non-stream request")
|
||||
else:
|
||||
include_usage = (
|
||||
isinstance(stream_options, dict) and stream_options.get("include_usage") is True
|
||||
)
|
||||
add(
|
||||
"stream_include_usage",
|
||||
"pass" if include_usage else "fail",
|
||||
"stream_options.include_usage is true"
|
||||
if include_usage
|
||||
else "stream_options.include_usage is missing or false",
|
||||
)
|
||||
|
||||
marker_count = _cache_marker_count(payload)
|
||||
if marker_count == 0:
|
||||
add("cache_marker_limit", "warn", "no cache markers found")
|
||||
else:
|
||||
add(
|
||||
"cache_marker_limit",
|
||||
"pass" if marker_count <= DASHSCOPE_CACHE_MARKER_LIMIT else "fail",
|
||||
f"cache markers={marker_count}, limit={DASHSCOPE_CACHE_MARKER_LIMIT}",
|
||||
)
|
||||
|
||||
paired, detail = _tool_call_pairing_ok(payload)
|
||||
add("tool_call_pairing", "pass" if paired else "fail", detail)
|
||||
|
||||
boolean_paths = _boolean_schema_paths(payload.get("tools") or [])
|
||||
add(
|
||||
"tool_schema_no_boolean_values",
|
||||
"pass" if not boolean_paths else "fail",
|
||||
"no boolean schema values"
|
||||
if not boolean_paths
|
||||
else "boolean schema values at " + ",".join(boolean_paths[:5]),
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def analyze_paths(paths: Iterable[Path]) -> tuple[dict[str, Any], list[dict[str, str]]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
checked_payloads = 0
|
||||
for path in paths:
|
||||
for wrapped in _iter_json_values(path):
|
||||
source_path = str(wrapped.get("__source_path") or path)
|
||||
for item in _payloads_from_value(source_path, wrapped.get("__value")):
|
||||
checked_payloads += 1
|
||||
rows.extend(
|
||||
_check_payload(
|
||||
item["source_path"],
|
||||
item["instance_id"],
|
||||
item["payload"],
|
||||
)
|
||||
)
|
||||
failures = Counter(row["check"] for row in rows if row["status"] == "fail")
|
||||
warnings = Counter(row["check"] for row in rows if row["status"] == "warn")
|
||||
summary = {
|
||||
"checked_payloads": checked_payloads,
|
||||
"rows": len(rows),
|
||||
"failed_checks": sum(failures.values()),
|
||||
"warning_checks": sum(warnings.values()),
|
||||
"failed_checks_by_name": dict(sorted(failures.items())),
|
||||
"warnings_by_name": dict(sorted(warnings.items())),
|
||||
}
|
||||
return summary, rows
|
||||
|
||||
|
||||
def write_outputs(
|
||||
summary: dict[str, Any],
|
||||
rows: list[dict[str, str]],
|
||||
*,
|
||||
json_path: Path,
|
||||
csv_path: Path,
|
||||
) -> None:
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fields = ["source_path", "instance_id", "model", "check", "status", "detail"]
|
||||
with csv_path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("paths", nargs="+", type=Path)
|
||||
parser.add_argument("--json-output", type=Path, default=Path("qwen_payload_parity.json"))
|
||||
parser.add_argument("--csv-output", type=Path, default=Path("qwen_payload_parity.csv"))
|
||||
args = parser.parse_args()
|
||||
summary, rows = analyze_paths(args.paths)
|
||||
write_outputs(summary, rows, json_path=args.json_output, csv_path=args.csv_output)
|
||||
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
||||
return 1 if summary["failed_checks"] else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Any
|
||||
|
||||
DASHSCOPE_DUPLICATE_MARKER = "duplicate tool interaction omitted"
|
||||
PROVIDER_COMPACTION_MARKER = "Historical tool call omitted for provider context budget"
|
||||
BARE_THINK_CLOSE_MARKER = "</think>"
|
||||
|
||||
|
||||
def _iter_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(row, dict):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _row_text(row: dict[str, Any]) -> str:
|
||||
return json.dumps(row, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def _numeric_timestamp(row: dict[str, Any]) -> float | None:
|
||||
for key in ("ts", "timestamp", "time", "created_at"):
|
||||
value = row.get(key)
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _instance_dirs(root: Path) -> list[Path]:
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted(path for path in root.iterdir() if path.is_dir())
|
||||
|
||||
|
||||
def _latency_summary(values: list[float]) -> dict[str, float | int | None]:
|
||||
if not values:
|
||||
return {"count": 0, "mean": None, "min": None, "max": None}
|
||||
ordered = sorted(values)
|
||||
return {
|
||||
"count": len(ordered),
|
||||
"mean": round(mean(ordered), 3),
|
||||
"min": round(ordered[0], 3),
|
||||
"max": round(ordered[-1], 3),
|
||||
}
|
||||
|
||||
|
||||
def _prediction_summary(predictions_path: Path | None) -> dict[str, Any]:
|
||||
rows = _iter_jsonl(predictions_path) if predictions_path is not None else []
|
||||
submitted = 0
|
||||
empty = 0
|
||||
instance_ids: set[str] = set()
|
||||
for row in rows:
|
||||
submitted += 1
|
||||
instance_id = row.get("instance_id")
|
||||
if isinstance(instance_id, str):
|
||||
instance_ids.add(instance_id)
|
||||
model_patch = row.get("model_patch")
|
||||
if not isinstance(model_patch, str) or not model_patch.strip():
|
||||
empty += 1
|
||||
return {
|
||||
"submitted": submitted,
|
||||
"unique_instance_ids": len(instance_ids),
|
||||
"empty_model_patch": empty,
|
||||
}
|
||||
|
||||
|
||||
def analyze_artifact_root(
|
||||
artifact_root: str | Path,
|
||||
*,
|
||||
predictions_path: str | Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
root = Path(artifact_root)
|
||||
prediction_path = Path(predictions_path) if predictions_path is not None else None
|
||||
instances = _instance_dirs(root)
|
||||
signals = {
|
||||
"dashscope_duplicate_omission": 0,
|
||||
"provider_compaction_omission": 0,
|
||||
"bare_think_close": 0,
|
||||
}
|
||||
patches = {"empty_git_patch": 0, "present_git_patch": 0}
|
||||
llm = {
|
||||
"requests": 0,
|
||||
"responses": 0,
|
||||
"response_chunks": 0,
|
||||
"errors": 0,
|
||||
"status_429": 0,
|
||||
"status_5xx": 0,
|
||||
"timeout_errors": 0,
|
||||
}
|
||||
request_ts_by_id: dict[str, float] = {}
|
||||
first_chunk_ts_by_id: dict[str, float] = {}
|
||||
|
||||
for instance_dir in instances:
|
||||
transcript_text = ""
|
||||
transcript_path = instance_dir / "transcript.jsonl"
|
||||
if transcript_path.exists():
|
||||
transcript_text = transcript_path.read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
signals["dashscope_duplicate_omission"] += transcript_text.count(
|
||||
DASHSCOPE_DUPLICATE_MARKER
|
||||
)
|
||||
signals["provider_compaction_omission"] += transcript_text.count(
|
||||
PROVIDER_COMPACTION_MARKER
|
||||
)
|
||||
signals["bare_think_close"] += transcript_text.count(BARE_THINK_CLOSE_MARKER)
|
||||
|
||||
patch_path = instance_dir / "git.patch"
|
||||
if patch_path.exists():
|
||||
patches["present_git_patch"] += 1
|
||||
if not patch_path.read_text(encoding="utf-8", errors="replace").strip():
|
||||
patches["empty_git_patch"] += 1
|
||||
|
||||
for row in _iter_jsonl(instance_dir / "llm_calls.jsonl"):
|
||||
event = str(row.get("event") or "")
|
||||
if event == "llm.request":
|
||||
llm["requests"] += 1
|
||||
request_id = row.get("request_id")
|
||||
ts = _numeric_timestamp(row)
|
||||
if isinstance(request_id, str) and ts is not None:
|
||||
request_ts_by_id.setdefault(request_id, ts)
|
||||
elif event == "llm.response":
|
||||
llm["responses"] += 1
|
||||
elif event == "llm.response_chunk":
|
||||
llm["response_chunks"] += 1
|
||||
request_id = row.get("request_id")
|
||||
ts = _numeric_timestamp(row)
|
||||
if isinstance(request_id, str) and ts is not None:
|
||||
first_chunk_ts_by_id.setdefault(request_id, ts)
|
||||
elif event == "llm.error":
|
||||
llm["errors"] += 1
|
||||
|
||||
status_code = row.get("status_code")
|
||||
if status_code == 429:
|
||||
llm["status_429"] += 1
|
||||
if isinstance(status_code, int) and 500 <= status_code <= 599:
|
||||
llm["status_5xx"] += 1
|
||||
text = _row_text(row).lower()
|
||||
if event == "llm.error" and "timeout" in text:
|
||||
llm["timeout_errors"] += 1
|
||||
|
||||
latencies = [
|
||||
first_ts - request_ts
|
||||
for request_id, request_ts in request_ts_by_id.items()
|
||||
if (first_ts := first_chunk_ts_by_id.get(request_id)) is not None
|
||||
and first_ts >= request_ts
|
||||
]
|
||||
llm["first_chunk_latency_seconds"] = _latency_summary(latencies)
|
||||
|
||||
return {
|
||||
"artifact_root": str(root),
|
||||
"instances": len(instances),
|
||||
"predictions": _prediction_summary(prediction_path),
|
||||
"patches": patches,
|
||||
"signals": signals,
|
||||
"llm": llm,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Analyze Qwen/DashScope provider-visible run artifact risks.",
|
||||
)
|
||||
parser.add_argument("artifact_root", type=Path)
|
||||
parser.add_argument("--predictions", type=Path, default=None)
|
||||
parser.add_argument("--pretty", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
summary = analyze_artifact_root(
|
||||
args.artifact_root,
|
||||
predictions_path=args.predictions,
|
||||
)
|
||||
indent = 2 if args.pretty else None
|
||||
print(json.dumps(summary, ensure_ascii=False, sort_keys=True, indent=indent))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify eval Docker image tags against a digest lock."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"lock file must contain a JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def _read_instance_ids(paths: list[Path]) -> list[str]:
|
||||
instance_ids: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
instance_id = raw_line.strip()
|
||||
if not instance_id or instance_id.startswith("#") or instance_id in seen:
|
||||
continue
|
||||
seen.add(instance_id)
|
||||
instance_ids.append(instance_id)
|
||||
return instance_ids
|
||||
|
||||
|
||||
def _records_by_instance(lock: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
records = lock.get("records")
|
||||
if not isinstance(records, list):
|
||||
raise ValueError("lock file is missing records[]")
|
||||
indexed: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError("lock records must be JSON objects")
|
||||
instance_id = record.get("instance_id")
|
||||
if not isinstance(instance_id, str) or not instance_id:
|
||||
raise ValueError("lock record missing instance_id")
|
||||
indexed[instance_id] = record
|
||||
return indexed
|
||||
|
||||
|
||||
def _inspect_image_id(image_ref: str) -> tuple[int, str, str]:
|
||||
proc = subprocess.run(
|
||||
["docker", "inspect", image_ref, "--format", "{{.Id}}"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return proc.returncode, proc.stdout.strip(), proc.stderr.strip()
|
||||
|
||||
|
||||
def _expected_instances(
|
||||
records: dict[str, dict[str, Any]],
|
||||
instance_files: list[Path],
|
||||
) -> list[str]:
|
||||
if instance_files:
|
||||
return _read_instance_ids(instance_files)
|
||||
return sorted(records)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--lock",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="Path to the image lock JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instance-file",
|
||||
action="append",
|
||||
default=[],
|
||||
type=Path,
|
||||
help="Instance id file to verify. May be repeated. Defaults to all lock records.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
lock = _read_json(args.lock)
|
||||
records = _records_by_instance(lock)
|
||||
instance_ids = _expected_instances(records, args.instance_file)
|
||||
except Exception as exc:
|
||||
print(f"invalid_lock: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
errors = 0
|
||||
for instance_id in instance_ids:
|
||||
record = records.get(instance_id)
|
||||
if record is None:
|
||||
print(f"missing_lock_record: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
image_ref = record.get("image_ref")
|
||||
expected_id = record.get("image_id")
|
||||
if not isinstance(image_ref, str) or not isinstance(expected_id, str):
|
||||
print(f"invalid_lock_record: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
returncode, actual_id, stderr = _inspect_image_id(image_ref)
|
||||
if returncode != 0:
|
||||
print(f"missing_image: {instance_id} {image_ref} {stderr}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
if actual_id != expected_id:
|
||||
print(
|
||||
f"digest_mismatch: {instance_id} {image_ref} "
|
||||
f"expected={expected_id} actual={actual_id}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f"checked={len(instance_ids)} errors={errors}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"checked={len(instance_ids)} errors=0 tag={lock.get('tag', '')}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assert LLM treatment delivery from per-instance llm_calls.jsonl records.
|
||||
|
||||
Experiment arms must verify that configured treatments actually reached the
|
||||
provider before a run is scored; any delivery mismatch makes the decision
|
||||
``invalid``. This scans every ``llm.request`` record in each instance's
|
||||
llm_calls.jsonl and asserts, on all requests:
|
||||
|
||||
- ``metadata.request_proof.proof_budget`` equals --expected-proof-budget
|
||||
- ``payload.reasoning.effort`` equals --expected-reasoning-effort
|
||||
(OpenRouter/GLM adapter: effort string; payloads carry no numeric budget)
|
||||
- ``payload.thinking_budget`` equals --expected-thinking-budget
|
||||
(DashScope/Qwen adapter: numeric budget)
|
||||
|
||||
The engine has a designed one-shot recovery that retries a failed provider
|
||||
call with thinking disabled; the request shape is adapter-specific
|
||||
(provider/openai.py): OpenRouter/GLM emits ``payload.reasoning = {"enabled":
|
||||
false}``; DashScope/Qwen emits ``payload.enable_thinking = false`` with no
|
||||
reasoning key and no thinking_budget. Such requests are counted separately
|
||||
and excluded from the reasoning-effort and thinking-budget assertions, but
|
||||
any occurrence beyond --allow-reasoning-fallbacks (default 0) is an error.
|
||||
The proof-budget assertion still applies to them. (Detection assumes a
|
||||
thinking-enabled arm; a deliberately thinking-off DashScope arm would count
|
||||
every request as a fallback.)
|
||||
|
||||
Separately, an httpx stream timeout with no stream event triggers a
|
||||
non-stream retry of the same budget-coordinated payload
|
||||
(``_complete_non_stream``); its record carries ``metadata.fallback_from ==
|
||||
"stream_timeout"`` and no ``request_proof`` block at all. These records are
|
||||
excluded from the proof-budget assertion (the payload assertions still
|
||||
apply), reported per instance as ``stream_fallbacks``, and never gated —
|
||||
the treatment itself was delivered unchanged.
|
||||
|
||||
One stdout line per instance reports the request count and distinct observed
|
||||
values. Exit is non-zero on any mismatch, unreadable request record, or
|
||||
instance with zero ``llm.request`` records.
|
||||
|
||||
Known limit: lines are prefiltered on the substring ``"llm.request"`` before
|
||||
JSON parsing, so a request line truncated within its first ~200 bytes (before
|
||||
the ``event`` key) is skipped silently rather than counted as unparsed. Tail
|
||||
truncation from a killed run cuts inside the large ``payload`` field instead,
|
||||
which still matches the prefilter and lands in the unparsed-error path — and a
|
||||
killed run is already invalid under the rc!=0 rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _instance_dirs(run_dir: Path) -> list[Path]:
|
||||
if (run_dir / "llm_calls.jsonl").is_file():
|
||||
return [run_dir]
|
||||
return sorted(child for child in run_dir.iterdir() if child.is_dir())
|
||||
|
||||
|
||||
class _InstanceScan:
|
||||
def __init__(self) -> None:
|
||||
self.requests = 0
|
||||
self.unparsed = 0
|
||||
self.reasoning_fallbacks = 0
|
||||
self.stream_fallbacks = 0
|
||||
self.proof_budgets: set[object] = set()
|
||||
self.efforts: set[object] = set()
|
||||
self.thinking_budgets: set[object] = set()
|
||||
|
||||
|
||||
def _scan_llm_requests(path: Path) -> _InstanceScan:
|
||||
scan = _InstanceScan()
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
for line in handle:
|
||||
if '"llm.request"' not in line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
scan.unparsed += 1
|
||||
continue
|
||||
if not isinstance(record, dict) or record.get("event") != "llm.request":
|
||||
continue
|
||||
scan.requests += 1
|
||||
metadata = record.get("metadata")
|
||||
metadata = metadata if isinstance(metadata, dict) else {}
|
||||
payload = record.get("payload")
|
||||
payload = payload if isinstance(payload, dict) else {}
|
||||
if metadata.get("fallback_from") == "stream_timeout":
|
||||
# Non-stream retry of a stream timeout re-sends the same
|
||||
# budget-coordinated payload, but its record carries no
|
||||
# request_proof metadata (provider/openai.py record_request).
|
||||
scan.stream_fallbacks += 1
|
||||
else:
|
||||
request_proof = metadata.get("request_proof")
|
||||
scan.proof_budgets.add(
|
||||
request_proof.get("proof_budget") if isinstance(request_proof, dict) else None
|
||||
)
|
||||
reasoning = payload.get("reasoning")
|
||||
if reasoning == {"enabled": False} or payload.get("enable_thinking") is False:
|
||||
# Exact shapes the engine's one-shot thinking-disable recovery
|
||||
# emits (provider/openai.py): OpenRouter reasoning={"enabled":
|
||||
# false}; DashScope enable_thinking=false with no reasoning key
|
||||
# and no thinking_budget. Anything else is a treatment value.
|
||||
scan.reasoning_fallbacks += 1
|
||||
else:
|
||||
scan.efforts.add(reasoning.get("effort") if isinstance(reasoning, dict) else None)
|
||||
scan.thinking_budgets.add(payload.get("thinking_budget"))
|
||||
return scan
|
||||
|
||||
|
||||
def _distinct(values: set[object]) -> str:
|
||||
return ",".join(sorted("null" if value is None else str(value) for value in values))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--run-dir",
|
||||
action="append",
|
||||
required=True,
|
||||
type=Path,
|
||||
help=(
|
||||
"Run directory holding per-instance subdirectories with llm_calls.jsonl "
|
||||
"(or a single instance directory). May be repeated."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-proof-budget",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Expected metadata.request_proof.proof_budget on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-reasoning-effort",
|
||||
default=None,
|
||||
help="Expected payload.reasoning.effort string on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-thinking-budget",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Expected numeric payload.thinking_budget on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-reasoning-fallbacks",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Max engine thinking-disable recovery requests tolerated per "
|
||||
"instance (OpenRouter payload.reasoning.enabled == false, or "
|
||||
"DashScope payload.enable_thinking == false)."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if (
|
||||
args.expected_proof_budget is None
|
||||
and args.expected_reasoning_effort is None
|
||||
and args.expected_thinking_budget is None
|
||||
):
|
||||
parser.error("at least one --expected-* assertion is required")
|
||||
|
||||
checked = 0
|
||||
errors = 0
|
||||
for run_dir in args.run_dir:
|
||||
if not run_dir.is_dir():
|
||||
print(f"missing_run_dir: {run_dir}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
instance_dirs = _instance_dirs(run_dir)
|
||||
if not instance_dirs:
|
||||
print(f"no_instances: {run_dir}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
for instance_dir in instance_dirs:
|
||||
checked += 1
|
||||
instance_id = instance_dir.name
|
||||
calls_path = instance_dir / "llm_calls.jsonl"
|
||||
if not calls_path.is_file():
|
||||
print(f"missing_llm_calls: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
scan = _scan_llm_requests(calls_path)
|
||||
print(
|
||||
f"instance={instance_id} requests={scan.requests} "
|
||||
f"proof_budget={_distinct(scan.proof_budgets)} "
|
||||
f"reasoning_effort={_distinct(scan.efforts)} "
|
||||
f"reasoning_fallbacks={scan.reasoning_fallbacks} "
|
||||
f"stream_fallbacks={scan.stream_fallbacks} "
|
||||
f"thinking_budget={_distinct(scan.thinking_budgets)}"
|
||||
)
|
||||
if scan.unparsed:
|
||||
print(
|
||||
f"unparsed_request_lines: {instance_id} count={scan.unparsed}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if scan.requests == 0:
|
||||
print(f"no_llm_requests: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
if scan.reasoning_fallbacks > args.allow_reasoning_fallbacks:
|
||||
print(
|
||||
f"reasoning_fallback_exceeded: {instance_id} "
|
||||
f"count={scan.reasoning_fallbacks} "
|
||||
f"allowed={args.allow_reasoning_fallbacks}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_proof_budget is not None and scan.proof_budgets != {
|
||||
args.expected_proof_budget
|
||||
}:
|
||||
print(
|
||||
f"proof_budget_mismatch: {instance_id} "
|
||||
f"expected={args.expected_proof_budget} "
|
||||
f"actual={_distinct(scan.proof_budgets)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_reasoning_effort is not None and scan.efforts != {
|
||||
args.expected_reasoning_effort
|
||||
}:
|
||||
print(
|
||||
f"reasoning_effort_mismatch: {instance_id} "
|
||||
f"expected={args.expected_reasoning_effort} "
|
||||
f"actual={_distinct(scan.efforts)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_thinking_budget is not None and scan.thinking_budgets != {
|
||||
args.expected_thinking_budget
|
||||
}:
|
||||
print(
|
||||
f"thinking_budget_mismatch: {instance_id} "
|
||||
f"expected={args.expected_thinking_budget} "
|
||||
f"actual={_distinct(scan.thinking_budgets)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f"checked={checked} errors={errors}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"checked={checked} errors=0")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,416 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for the OpenSquilla experiment ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_LEDGER_ROOT = Path("./.experiments-ledger")
|
||||
RUNNER_RELATIVE_PATH = Path("scripts/run_tool_policy_validation_stdin_keys.sh")
|
||||
|
||||
# Docker needles for run supervision; overridable per run via the top-level
|
||||
# manifest keys ``container_name_prefix`` and ``eval_image_needle``.
|
||||
CONTAINER_NAME_PREFIX = "opensquilla-swe-"
|
||||
EVAL_IMAGE_NEEDLE = "sweb.eval."
|
||||
|
||||
# Default provider secret env var required per model family; overridable via
|
||||
# the ``--required-secret-env`` CLI flag on exp_init.
|
||||
DEFAULT_REQUIRED_SECRET_ENV = {
|
||||
"qwen": "DASHSCOPE_API_KEY",
|
||||
"glm": "OPENROUTER_API_KEY",
|
||||
}
|
||||
|
||||
EXP_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
|
||||
SECRET_KEY_RE = re.compile(r"(api[_-]?key|token|secret|password|credential)", re.I)
|
||||
TRACKED_ENV_PREFIXES = ("OPENSQUILLA_",)
|
||||
|
||||
|
||||
class LedgerError(RuntimeError):
|
||||
"""Raised for user-correctable ledger command failures."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitInfo:
|
||||
path: str
|
||||
branch: str
|
||||
head: str
|
||||
short_head: str
|
||||
dirty_count: int
|
||||
dirty_summary: list[str]
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(UTC).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def ledger_root_from_env() -> Path:
|
||||
return Path(
|
||||
os.environ.get(
|
||||
"OPENSQUILLA_EXPERIMENT_LEDGER_ROOT",
|
||||
os.environ.get("OPENSQUILLA_SWE_EXPERIMENT_LEDGER_ROOT", DEFAULT_LEDGER_ROOT),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def validate_exp_id(exp_id: str) -> str:
|
||||
if not EXP_ID_RE.fullmatch(exp_id):
|
||||
raise LedgerError("exp_id must match [a-z0-9][a-z0-9._-]*")
|
||||
return exp_id
|
||||
|
||||
|
||||
def ensure_ledger_layout(root: Path) -> None:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
(root / "runs").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def ledger_lock(root: Path) -> Iterator[None]:
|
||||
ensure_ledger_layout(root)
|
||||
lock_path = root / ".lock"
|
||||
with lock_path.open("a", encoding="utf-8") as handle:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
dir=str(path.parent),
|
||||
prefix=f".{path.name}.",
|
||||
delete=False,
|
||||
) as handle:
|
||||
handle.write(text)
|
||||
tmp = Path(handle.name)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
atomic_write_text(path, json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def read_json(path: Path, default: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {} if default is None else dict(default)
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {} if default is None else dict(default)
|
||||
return data if isinstance(data, dict) else ({} if default is None else dict(default))
|
||||
|
||||
|
||||
def read_json_strict(path: Path, *, label: str = "JSON file") -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise LedgerError(f"missing {label}: {path}")
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise LedgerError(f"invalid {label}: {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise LedgerError(f"{label} must contain a JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def append_jsonl(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def run_command(args: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=False)
|
||||
|
||||
|
||||
def git_info(path: Path, *, max_dirty_lines: int = 20) -> GitInfo:
|
||||
if not path.exists():
|
||||
raise LedgerError(f"path does not exist: {path}")
|
||||
head = _git_stdout(path, ["rev-parse", "HEAD"])
|
||||
short_head = _git_stdout(path, ["rev-parse", "--short", "HEAD"])
|
||||
branch_proc = run_command(["git", "branch", "--show-current"], cwd=path)
|
||||
branch = branch_proc.stdout.strip() if branch_proc.returncode == 0 else ""
|
||||
dirty_proc = run_command(["git", "status", "--short"], cwd=path)
|
||||
if dirty_proc.returncode != 0:
|
||||
raise LedgerError(f"git status failed for {path}: {dirty_proc.stderr.strip()}")
|
||||
dirty_lines = [line for line in dirty_proc.stdout.splitlines() if line.strip()]
|
||||
return GitInfo(
|
||||
path=str(path),
|
||||
branch=branch,
|
||||
head=head,
|
||||
short_head=short_head,
|
||||
dirty_count=len(dirty_lines),
|
||||
dirty_summary=dirty_lines[:max_dirty_lines],
|
||||
)
|
||||
|
||||
|
||||
def _git_stdout(path: Path, args: list[str]) -> str:
|
||||
proc = run_command(["git", *args], cwd=path)
|
||||
if proc.returncode != 0:
|
||||
raise LedgerError(f"git {' '.join(args)} failed for {path}: {proc.stderr.strip()}")
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def copy_snapshot(src: Path, dst_dir: Path) -> dict[str, str]:
|
||||
if not src.is_file():
|
||||
raise LedgerError(f"snapshot source must be a file: {src}")
|
||||
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||
dst = dst_dir / src.name
|
||||
shutil.copy2(src, dst)
|
||||
return {"source": str(src), "snapshot": str(dst), "sha256": sha256_file(src)}
|
||||
|
||||
|
||||
def parse_key_value_file(path: Path) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return result
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
result[key.strip()] = value.strip()
|
||||
return result
|
||||
|
||||
|
||||
def parse_env_overrides(items: list[str]) -> dict[str, dict[str, Any]]:
|
||||
parsed: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if "=" not in item:
|
||||
raise LedgerError(f"--env must use KEY=VALUE form: {item}")
|
||||
key, value = item.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key:
|
||||
raise LedgerError("--env key cannot be empty")
|
||||
parsed[key] = redact_env_value(key, value)
|
||||
return parsed
|
||||
|
||||
|
||||
def redact_env_value(key: str, value: str | None = None) -> dict[str, Any]:
|
||||
if SECRET_KEY_RE.search(key):
|
||||
return {"required": True, "provided_at_init": bool(value), "redacted": True}
|
||||
if key.startswith(TRACKED_ENV_PREFIXES):
|
||||
return {"value": value or "", "redacted": False}
|
||||
return {"value": value or "", "redacted": False}
|
||||
|
||||
|
||||
def env_exports_for_command(env: dict[str, dict[str, Any]]) -> list[str]:
|
||||
exports: list[str] = []
|
||||
for key, meta in sorted(env.items()):
|
||||
if meta.get("redacted"):
|
||||
continue
|
||||
value = str(meta.get("value", ""))
|
||||
exports.append(f"export {key}={sh_quote(value)}")
|
||||
return exports
|
||||
|
||||
|
||||
def sh_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
def required_secret_env(
|
||||
run_mode: str, mapping: dict[str, str] | None = None
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
if mapping is None:
|
||||
mapping = DEFAULT_REQUIRED_SECRET_ENV
|
||||
required: dict[str, dict[str, Any]] = {}
|
||||
if run_mode != "glm_only":
|
||||
key = mapping["qwen"]
|
||||
required[key] = redact_env_value(key)
|
||||
if run_mode != "qwen_only":
|
||||
key = mapping["glm"]
|
||||
required[key] = redact_env_value(key)
|
||||
return required
|
||||
|
||||
|
||||
def read_first_existing_report(paths_file: Path) -> list[str]:
|
||||
if not paths_file.exists():
|
||||
return []
|
||||
reports = []
|
||||
for line in paths_file.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
candidate = line.strip()
|
||||
if candidate and Path(candidate).is_file():
|
||||
reports.append(candidate)
|
||||
return reports
|
||||
|
||||
|
||||
def collect_eval_metrics(report_paths: list[str]) -> dict[str, Any]:
|
||||
totals = {
|
||||
"total_instances": 0,
|
||||
"submitted_instances": 0,
|
||||
"completed_instances": 0,
|
||||
"resolved_instances": 0,
|
||||
"unresolved_instances": 0,
|
||||
"empty_patch_instances": 0,
|
||||
"error_instances": 0,
|
||||
}
|
||||
resolved_ids: list[str] = []
|
||||
empty_patch_ids: list[str] = []
|
||||
error_ids: list[str] = []
|
||||
reports: list[dict[str, Any]] = []
|
||||
for path_str in report_paths:
|
||||
path = Path(path_str)
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
for key in totals:
|
||||
value = data.get(key)
|
||||
if isinstance(value, int):
|
||||
totals[key] += value
|
||||
resolved_ids.extend(_string_list(data.get("resolved_ids")))
|
||||
empty_patch_ids.extend(_string_list(data.get("empty_patch_ids")))
|
||||
error_ids.extend(_string_list(data.get("error_ids")))
|
||||
reports.append({"path": str(path), "schema_version": data.get("schema_version", "")})
|
||||
return {
|
||||
**totals,
|
||||
"resolved_ids": sorted(set(resolved_ids)),
|
||||
"empty_patch_ids": sorted(set(empty_patch_ids)),
|
||||
"error_ids": sorted(set(error_ids)),
|
||||
"report_count": len(reports),
|
||||
"reports": reports,
|
||||
}
|
||||
|
||||
|
||||
def _string_list(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def active_processes() -> list[dict[str, Any]]:
|
||||
proc = run_command(["ps", "-eo", "pid=,args="])
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
needles = (
|
||||
"run_tool_policy_validation",
|
||||
"run_infer.py",
|
||||
"run_eval.py",
|
||||
"swebench.harness.run_evaluation",
|
||||
)
|
||||
rows = []
|
||||
self_pid = os.getpid()
|
||||
for line in proc.stdout.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
pid_text, _, args = stripped.partition(" ")
|
||||
try:
|
||||
pid = int(pid_text)
|
||||
except ValueError:
|
||||
continue
|
||||
if pid == self_pid:
|
||||
continue
|
||||
if any(needle in args for needle in needles):
|
||||
rows.append({"pid": pid, "args": args[:500]})
|
||||
return rows
|
||||
|
||||
|
||||
def active_swe_containers(manifest: dict[str, Any] | None = None) -> list[str]:
|
||||
config = manifest if isinstance(manifest, dict) else {}
|
||||
container_prefix = str(config.get("container_name_prefix") or CONTAINER_NAME_PREFIX)
|
||||
eval_image_needle = str(config.get("eval_image_needle") or EVAL_IMAGE_NEEDLE)
|
||||
proc = run_command(["docker", "ps", "--format", "{{.Names}}"])
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
return [
|
||||
line.strip()
|
||||
for line in proc.stdout.splitlines()
|
||||
if line.startswith(container_prefix) or line.startswith(eval_image_needle)
|
||||
]
|
||||
|
||||
|
||||
def status_label_from_dirty(info: GitInfo) -> str:
|
||||
return "clean" if info.dirty_count == 0 else f"dirty {info.dirty_count}"
|
||||
|
||||
|
||||
def exp_dir(root: Path, exp_id: str) -> Path:
|
||||
return root / "runs" / validate_exp_id(exp_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Contamination registry (quarantined runs)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# ``contaminations.json`` at the ledger root maps a contamination class (e.g.
|
||||
# ``tool_result_compaction_defect``) to the artifact batch names whose
|
||||
# results are confounded by an infra defect. Baseline and A/B tooling must
|
||||
# exclude quarantined artifacts; ``exp_status`` warns when a recorded
|
||||
# baseline references one.
|
||||
|
||||
CONTAMINATIONS_FILENAME = "contaminations.json"
|
||||
|
||||
|
||||
def contaminations_path(root: Path) -> Path:
|
||||
return root / CONTAMINATIONS_FILENAME
|
||||
|
||||
|
||||
def load_contaminations(root: Path) -> dict[str, Any]:
|
||||
data = read_json(contaminations_path(root), default={})
|
||||
classes = data.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
data["classes"] = {}
|
||||
data.setdefault("version", 1)
|
||||
return data
|
||||
|
||||
|
||||
def artifact_basename(artifact: str | Path) -> str:
|
||||
"""Normalize an artifact path or batch name to its bare batch name."""
|
||||
return str(artifact).rstrip("/").rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def contamination_class_for(
|
||||
root: Path,
|
||||
artifact: str | Path,
|
||||
contaminations: dict[str, Any] | None = None,
|
||||
) -> str | None:
|
||||
"""Return the contamination class covering ``artifact``, or None if clean."""
|
||||
name = artifact_basename(artifact)
|
||||
if not name:
|
||||
return None
|
||||
data = contaminations if contaminations is not None else load_contaminations(root)
|
||||
classes = data.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
return None
|
||||
for class_name, payload in sorted(classes.items()):
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
names = payload.get("artifact_names")
|
||||
if isinstance(names, list) and name in names:
|
||||
return class_name
|
||||
return None
|
||||
|
||||
|
||||
def run_dir_contamination_classes(root: Path, exp_id: str) -> list[str]:
|
||||
"""Return contamination classes stamped on a ledger run dir (empty if clean)."""
|
||||
if not exp_id or not EXP_ID_RE.fullmatch(exp_id):
|
||||
return []
|
||||
stamp = read_json(root / "runs" / exp_id / "contamination.json")
|
||||
classes = stamp.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
return []
|
||||
return sorted(name for name in classes if isinstance(name, str))
|
||||
@@ -0,0 +1,684 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Finalize an OpenSquilla experiment from handoff artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
atomic_write_text,
|
||||
collect_eval_metrics,
|
||||
contamination_class_for,
|
||||
exp_dir,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
parse_key_value_file,
|
||||
read_first_existing_report,
|
||||
read_json,
|
||||
read_json_strict,
|
||||
run_dir_contamination_classes,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
DECISIONS = {"adopted", "rejected", "observe", "inconclusive", "invalid", "stopped"}
|
||||
NO_VALID_EVAL_DECISIONS = {"invalid", "stopped"}
|
||||
AGENT_ENV_DELIVERY_VARS = frozenset(
|
||||
{
|
||||
"OPENSQUILLA_DASHSCOPE_THINKING_BUDGET",
|
||||
"OPENSQUILLA_FINAL_DIFF_CONTRACT_MODE",
|
||||
"OPENSQUILLA_FINALIZE_EVIDENCE_GATE",
|
||||
"OPENSQUILLA_PATCH_EVIDENCE_PROTOCOL",
|
||||
"OPENSQUILLA_PROVIDER_COMPACTION_TINY_GUARD_CHARS",
|
||||
"OPENSQUILLA_PROVIDER_COMPACTION_PROTECT_RECENT_ASSISTANT",
|
||||
"OPENSQUILLA_PROVIDER_CONTEXT_BLOCK_FEEDBACK",
|
||||
"OPENSQUILLA_IDENTICAL_REQUEST_LOOP_BREAK",
|
||||
"OPENSQUILLA_PLACEHOLDER_ESCALATION_THRESHOLD",
|
||||
"OPENSQUILLA_DEADLINE_WRAPUP_MARGIN_SECONDS",
|
||||
"OPENSQUILLA_TOOL_REPEAT_NUDGE_THRESHOLD",
|
||||
"OPENSQUILLA_TOOL_REPEAT_NUDGE_TOOLS",
|
||||
"OPENSQUILLA_PROVIDER_HISTORY_DEDUP",
|
||||
"OPENSQUILLA_PROVIDER_HISTORY_DEDUP_MIN_REPEATS",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
parser.add_argument("--batch-dir", type=Path, required=True)
|
||||
parser.add_argument("--decision", required=True, choices=sorted(DECISIONS))
|
||||
parser.add_argument("--decision-reason", required=True)
|
||||
parser.add_argument("--mechanism", default="")
|
||||
parser.add_argument("--baseline-model", choices=["qwen", "glm"], default="")
|
||||
parser.add_argument("--overwrite-decision", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
finalize(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def finalize(args: argparse.Namespace) -> None:
|
||||
exp_id = validate_exp_id(args.exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
manifest = read_json_strict(run_dir / "manifest.json", label="experiment manifest")
|
||||
if (run_dir / "decision.md").exists() and not args.overwrite_decision:
|
||||
raise LedgerError("decision already exists; pass --overwrite-decision to replace")
|
||||
if not args.batch_dir.is_dir():
|
||||
raise LedgerError(f"batch dir does not exist: {args.batch_dir}")
|
||||
|
||||
artifacts = collect_artifacts(args.batch_dir)
|
||||
validate_batch_matches_manifest(manifest, artifacts)
|
||||
artifacts["env_delivery"] = collect_env_delivery(manifest, artifacts)
|
||||
metrics = collect_metrics(artifacts)
|
||||
eval_valid = (
|
||||
metrics["eval_report_count"] > 0
|
||||
and not metrics["nonzero_eval_exit_codes"]
|
||||
and not metrics["nonzero_infer_exit_codes"]
|
||||
and not metrics["nonzero_other_exit_codes"]
|
||||
)
|
||||
if not eval_valid and args.decision not in NO_VALID_EVAL_DECISIONS:
|
||||
raise LedgerError(
|
||||
"missing/nonzero infer or eval results can only be finalized as invalid or stopped"
|
||||
)
|
||||
env_delivery_errors = artifacts.get("env_delivery", {}).get("errors", [])
|
||||
if env_delivery_errors and args.decision not in NO_VALID_EVAL_DECISIONS:
|
||||
raise LedgerError(
|
||||
"manifest-pinned runtime env was not delivered to agent instances: "
|
||||
+ "; ".join(str(error) for error in env_delivery_errors)
|
||||
)
|
||||
if args.decision == "adopted" and args.baseline_model:
|
||||
contamination_classes = _batch_contamination_classes(ledger_root, exp_id, artifacts)
|
||||
if contamination_classes:
|
||||
raise LedgerError(
|
||||
"cannot adopt as baseline: batch is quarantined ("
|
||||
+ ", ".join(contamination_classes)
|
||||
+ "); re-baseline on clean runs"
|
||||
)
|
||||
|
||||
finished_at = now_iso()
|
||||
decision_payload = {
|
||||
"exp_id": exp_id,
|
||||
"decision": args.decision,
|
||||
"reason": args.decision_reason,
|
||||
"mechanism": args.mechanism,
|
||||
"baseline_model": args.baseline_model,
|
||||
"decided_at": finished_at,
|
||||
}
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(run_dir / "artifacts.json", artifacts)
|
||||
atomic_write_json(run_dir / "metrics.json", metrics)
|
||||
atomic_write_text(run_dir / "analysis.md", render_analysis(manifest, artifacts, metrics))
|
||||
atomic_write_text(run_dir / "decision.md", render_decision(decision_payload, metrics))
|
||||
update_current(ledger_root, exp_id, args.decision, metrics, finished_at)
|
||||
if args.decision == "adopted" and args.baseline_model:
|
||||
update_baseline(
|
||||
ledger_root,
|
||||
args.baseline_model,
|
||||
exp_id,
|
||||
manifest,
|
||||
metrics,
|
||||
args.decision_reason,
|
||||
)
|
||||
if args.mechanism:
|
||||
update_mechanism(
|
||||
ledger_root,
|
||||
args.mechanism,
|
||||
args.decision,
|
||||
exp_id,
|
||||
args.decision_reason,
|
||||
)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": finished_at,
|
||||
"exp_id": exp_id,
|
||||
"event": "finalized",
|
||||
"decision": args.decision,
|
||||
"resolved": metrics["resolved_instances"],
|
||||
"total": metrics["total_instances"],
|
||||
"empty": metrics["empty_patch_instances"],
|
||||
"batch_dir": str(args.batch_dir),
|
||||
},
|
||||
)
|
||||
print(json.dumps({"exp_id": exp_id, "decision": args.decision, "metrics": metrics}, indent=2))
|
||||
|
||||
|
||||
def _batch_contamination_classes(
|
||||
ledger_root: Path, exp_id: str, artifacts: dict[str, Any]
|
||||
) -> list[str]:
|
||||
"""Return contamination classes covering this batch, by name or by stamped run dir."""
|
||||
classes = set(run_dir_contamination_classes(ledger_root, exp_id))
|
||||
batch_dir = str(artifacts.get("batch_dir") or "").strip()
|
||||
if batch_dir:
|
||||
batch_class = contamination_class_for(ledger_root, batch_dir)
|
||||
if batch_class:
|
||||
classes.add(batch_class)
|
||||
return sorted(classes)
|
||||
|
||||
|
||||
def collect_artifacts(batch_dir: Path) -> dict[str, Any]:
|
||||
manifest_txt = batch_dir / "manifest.txt"
|
||||
batch_manifest = parse_key_value_file(manifest_txt)
|
||||
report_paths: list[str] = []
|
||||
for path_file in sorted(batch_dir.glob("*-eval.report_paths.txt")):
|
||||
report_paths.extend(read_first_existing_report(path_file))
|
||||
exit_codes = {}
|
||||
for exit_file in sorted(batch_dir.glob("*.exit_code")):
|
||||
text = exit_file.read_text(encoding="utf-8", errors="replace").strip()
|
||||
try:
|
||||
value: int | str = int(text)
|
||||
except ValueError:
|
||||
value = text
|
||||
exit_codes[exit_file.name] = value
|
||||
return {
|
||||
"batch_dir": str(batch_dir),
|
||||
"batch_manifest_path": str(manifest_txt) if manifest_txt.exists() else "",
|
||||
"batch_manifest": batch_manifest,
|
||||
"eval_report_paths": sorted(set(report_paths)),
|
||||
"exit_codes": exit_codes,
|
||||
"supervisor_log": str(batch_dir / "supervisor.log")
|
||||
if (batch_dir / "supervisor.log").exists()
|
||||
else "",
|
||||
}
|
||||
|
||||
|
||||
def collect_env_delivery(
|
||||
manifest: dict[str, Any],
|
||||
artifacts: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
expected = _manifest_agent_env_expectations(manifest)
|
||||
delivery: dict[str, Any] = {
|
||||
"expected": expected,
|
||||
"checked_instance_count": 0,
|
||||
"run_dirs": [],
|
||||
"per_var": {
|
||||
name: {
|
||||
"expected": value,
|
||||
"matched": 0,
|
||||
"missing": 0,
|
||||
"mismatch": 0,
|
||||
}
|
||||
for name, value in expected.items()
|
||||
},
|
||||
"errors": [],
|
||||
}
|
||||
if not expected:
|
||||
return delivery
|
||||
|
||||
run_dirs = _agent_run_dirs(artifacts)
|
||||
delivery["run_dirs"] = [str(path) for path in run_dirs]
|
||||
if not run_dirs:
|
||||
delivery["errors"].append(
|
||||
"no agent run dirs resolved from batch manifest; env delivery could not be "
|
||||
"verified for expected vars: " + ", ".join(sorted(expected))
|
||||
)
|
||||
return delivery
|
||||
|
||||
metadata_paths: list[Path] = []
|
||||
for run_dir in run_dirs:
|
||||
if not run_dir.is_dir():
|
||||
delivery["errors"].append(f"run artifact dir not found: {run_dir}")
|
||||
continue
|
||||
metadata_paths.extend(sorted(run_dir.glob("*/metadata.json")))
|
||||
if not metadata_paths:
|
||||
delivery["errors"].append("no instance metadata found for env delivery check")
|
||||
return delivery
|
||||
|
||||
for metadata_path in metadata_paths:
|
||||
delivery["checked_instance_count"] += 1
|
||||
try:
|
||||
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError):
|
||||
delivery["errors"].append(f"metadata is not valid JSON: {metadata_path}")
|
||||
continue
|
||||
forwarded_env = (
|
||||
((payload.get("agent") or {}).get("controls") or {}).get(
|
||||
"progress_watchdog_env"
|
||||
)
|
||||
or {}
|
||||
)
|
||||
if not isinstance(forwarded_env, dict):
|
||||
delivery["errors"].append(
|
||||
f"progress_watchdog_env is not an object: {metadata_path}"
|
||||
)
|
||||
forwarded_env = {}
|
||||
for name, expected_value in expected.items():
|
||||
stats = delivery["per_var"][name]
|
||||
if name not in forwarded_env:
|
||||
stats["missing"] += 1
|
||||
elif str(forwarded_env.get(name)) != expected_value:
|
||||
stats["mismatch"] += 1
|
||||
else:
|
||||
stats["matched"] += 1
|
||||
|
||||
for name, stats in delivery["per_var"].items():
|
||||
missing = int(stats["missing"])
|
||||
mismatch = int(stats["mismatch"])
|
||||
if missing or mismatch:
|
||||
delivery["errors"].append(
|
||||
f"{name} expected {stats['expected']!r}: "
|
||||
f"{missing} missing, {mismatch} mismatched across "
|
||||
f"{delivery['checked_instance_count']} metadata files"
|
||||
)
|
||||
return delivery
|
||||
|
||||
|
||||
def _manifest_agent_env_expectations(manifest: dict[str, Any]) -> dict[str, str]:
|
||||
env = manifest.get("config", {}).get("env", {})
|
||||
if not isinstance(env, dict):
|
||||
return {}
|
||||
expected: dict[str, str] = {}
|
||||
for name, payload in env.items():
|
||||
if name not in AGENT_ENV_DELIVERY_VARS:
|
||||
continue
|
||||
if not isinstance(payload, dict) or payload.get("redacted"):
|
||||
continue
|
||||
value = payload.get("value")
|
||||
if value is not None:
|
||||
expected[name] = str(value)
|
||||
return expected
|
||||
|
||||
|
||||
def _agent_run_dirs(artifacts: dict[str, Any]) -> list[Path]:
|
||||
batch_dir_text = str(artifacts.get("batch_dir") or "").strip()
|
||||
if not batch_dir_text:
|
||||
return []
|
||||
batch_dir = Path(batch_dir_text)
|
||||
batch_manifest = artifacts.get("batch_manifest") or {}
|
||||
run_mode = str(batch_manifest.get("run_mode") or "")
|
||||
keys: list[tuple[str, str]] = []
|
||||
if run_mode != "glm_only":
|
||||
keys.extend(
|
||||
[
|
||||
("qwen_ml_run_id", "ml_ids"),
|
||||
("qwen_verified_run_id", "verified_ids"),
|
||||
]
|
||||
)
|
||||
if run_mode != "qwen_only":
|
||||
keys.extend(
|
||||
[
|
||||
("glm_ml_run_id", "ml_ids"),
|
||||
("glm_verified_run_id", "verified_ids"),
|
||||
]
|
||||
)
|
||||
roots: list[Path] = []
|
||||
for key, ids_key in keys:
|
||||
if not str(batch_manifest.get(ids_key) or "").strip():
|
||||
continue
|
||||
run_id = str(batch_manifest.get(key) or "")
|
||||
if run_id:
|
||||
roots.append(batch_dir.parent / run_id)
|
||||
return roots
|
||||
|
||||
|
||||
def validate_batch_matches_manifest(manifest: dict[str, Any], artifacts: dict[str, Any]) -> None:
|
||||
batch_manifest = artifacts.get("batch_manifest", {})
|
||||
if not batch_manifest:
|
||||
raise LedgerError("batch manifest.txt is missing or empty")
|
||||
|
||||
errors: list[str] = []
|
||||
_expect_equal(
|
||||
errors,
|
||||
"opensquilla_source_head",
|
||||
batch_manifest.get("opensquilla_source_head"),
|
||||
manifest.get("source", {}).get("head"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"handoff_head",
|
||||
batch_manifest.get("handoff_head"),
|
||||
manifest.get("handoff", {}).get("head"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"condition_label",
|
||||
batch_manifest.get("condition_label"),
|
||||
manifest.get("config", {}).get("condition_label"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"run_mode",
|
||||
batch_manifest.get("run_mode"),
|
||||
manifest.get("execution", {}).get("run_mode"),
|
||||
)
|
||||
for key in ("qwen_workers", "glm_workers", "eval_workers"):
|
||||
_expect_equal(
|
||||
errors,
|
||||
key,
|
||||
batch_manifest.get(key),
|
||||
str(manifest.get("execution", {}).get(key, "")),
|
||||
)
|
||||
|
||||
run_mode = str(manifest.get("execution", {}).get("run_mode", ""))
|
||||
if run_mode != "glm_only":
|
||||
_expect_equal(
|
||||
errors,
|
||||
"qwen_config_sha256",
|
||||
batch_manifest.get("qwen_config_sha256"),
|
||||
manifest.get("config", {}).get("qwen_config", {}).get("sha256"),
|
||||
)
|
||||
if run_mode != "qwen_only":
|
||||
_expect_equal(
|
||||
errors,
|
||||
"glm_config_sha256",
|
||||
batch_manifest.get("glm_config_sha256"),
|
||||
manifest.get("config", {}).get("glm_config", {}).get("sha256"),
|
||||
)
|
||||
|
||||
_expect_one_of(
|
||||
errors,
|
||||
"ml_instance_file",
|
||||
batch_manifest.get("ml_instance_file"),
|
||||
_path_candidates(manifest.get("slice", {}).get("ml", {})),
|
||||
)
|
||||
_expect_one_of(
|
||||
errors,
|
||||
"verified_instance_file",
|
||||
batch_manifest.get("verified_instance_file"),
|
||||
_path_candidates(manifest.get("slice", {}).get("verified", {})),
|
||||
)
|
||||
|
||||
_verify_slice_content(
|
||||
errors,
|
||||
"ml_instance_file",
|
||||
batch_manifest.get("ml_instance_file"),
|
||||
manifest.get("slice", {}).get("ml", {}),
|
||||
)
|
||||
_verify_slice_content(
|
||||
errors,
|
||||
"verified_instance_file",
|
||||
batch_manifest.get("verified_instance_file"),
|
||||
manifest.get("slice", {}).get("verified", {}),
|
||||
)
|
||||
_verify_batch_selected_id_count(
|
||||
errors,
|
||||
"ml_ids",
|
||||
batch_manifest.get("ml_ids"),
|
||||
manifest.get("slice", {}).get("ml", {}),
|
||||
)
|
||||
_verify_batch_selected_id_count(
|
||||
errors,
|
||||
"verified_ids",
|
||||
batch_manifest.get("verified_ids"),
|
||||
manifest.get("slice", {}).get("verified", {}),
|
||||
)
|
||||
|
||||
_verify_runner_sha(errors, batch_manifest, manifest)
|
||||
|
||||
if errors:
|
||||
raise LedgerError("batch does not match experiment manifest: " + "; ".join(errors))
|
||||
|
||||
|
||||
def _expect_equal(errors: list[str], label: str, actual: str | None, expected: Any) -> None:
|
||||
expected_text = "" if expected is None else str(expected)
|
||||
actual_text = "" if actual is None else str(actual)
|
||||
if not actual_text:
|
||||
errors.append(f"{label} missing")
|
||||
elif actual_text != expected_text:
|
||||
errors.append(f"{label}={actual_text!r} expected {expected_text!r}")
|
||||
|
||||
|
||||
def _expect_one_of(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual: str | None,
|
||||
expected_values: set[str],
|
||||
) -> None:
|
||||
actual_text = "" if actual is None else str(actual)
|
||||
if not actual_text:
|
||||
errors.append(f"{label} missing")
|
||||
elif actual_text not in expected_values:
|
||||
expected = ", ".join(sorted(expected_values))
|
||||
errors.append(f"{label}={actual_text!r} expected one of [{expected}]")
|
||||
|
||||
|
||||
def _path_candidates(payload: dict[str, Any]) -> set[str]:
|
||||
return {str(payload.get(key)) for key in ("source", "snapshot") if payload.get(key)}
|
||||
|
||||
|
||||
def _verify_slice_content(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual_path: str | None,
|
||||
slice_payload: dict[str, Any],
|
||||
) -> None:
|
||||
"""Confirm the batch's instance file matches the manifest by content, not just path.
|
||||
|
||||
The batch manifest.txt records only the instance-file path; a path match alone does
|
||||
not prove the file's contents (or slice size) are the ones the manifest pinned.
|
||||
"""
|
||||
path_text = "" if actual_path is None else str(actual_path)
|
||||
if not path_text:
|
||||
return # missing path already reported by _expect_one_of
|
||||
path = Path(path_text)
|
||||
if not path.is_file():
|
||||
errors.append(f"{label} not readable for content check: {path}")
|
||||
return
|
||||
expected_sha = str(slice_payload.get("sha256") or "")
|
||||
if expected_sha and sha256_file(path) != expected_sha:
|
||||
errors.append(f"{label} sha256 changed since manifest creation")
|
||||
expected_count = slice_payload.get("count")
|
||||
if isinstance(expected_count, int):
|
||||
if expected_count == 0:
|
||||
return
|
||||
actual_count = _count_instances(path)
|
||||
if actual_count != expected_count:
|
||||
errors.append(
|
||||
f"{label} instance count {actual_count} != manifest {expected_count}"
|
||||
)
|
||||
|
||||
|
||||
def _count_instances(path: Path) -> int:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
return sum(1 for line in text.splitlines() if line.strip())
|
||||
|
||||
|
||||
def _verify_batch_selected_id_count(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual_ids: str | None,
|
||||
slice_payload: dict[str, Any],
|
||||
) -> None:
|
||||
expected_count = slice_payload.get("count")
|
||||
if not isinstance(expected_count, int):
|
||||
return
|
||||
actual = 0 if actual_ids is None else len(str(actual_ids).split())
|
||||
if actual != expected_count:
|
||||
errors.append(f"{label} selected count {actual} != manifest {expected_count}")
|
||||
|
||||
|
||||
def _verify_runner_sha(
|
||||
errors: list[str],
|
||||
batch_manifest: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Opportunistically confirm the batch was produced by the manifest-pinned runner.
|
||||
|
||||
The handoff runner does not emit a runner sha into manifest.txt today, so this is a
|
||||
no-op unless a ``runner_sha256``/``handoff_runner_sha256`` field is present. Runner
|
||||
integrity at launch time is enforced separately in exp_run.py.
|
||||
"""
|
||||
expected = str(manifest.get("config", {}).get("runner", {}).get("sha256") or "")
|
||||
if not expected:
|
||||
return
|
||||
actual = (
|
||||
batch_manifest.get("runner_sha256")
|
||||
or batch_manifest.get("handoff_runner_sha256")
|
||||
or ""
|
||||
)
|
||||
actual_text = str(actual)
|
||||
if actual_text and actual_text != expected:
|
||||
errors.append("runner_sha256 does not match manifest runner")
|
||||
|
||||
|
||||
def collect_metrics(artifacts: dict[str, Any]) -> dict[str, Any]:
|
||||
eval_metrics = collect_eval_metrics(artifacts.get("eval_report_paths", []))
|
||||
exit_codes = artifacts.get("exit_codes", {})
|
||||
nonzero = {name: value for name, value in exit_codes.items() if value != 0}
|
||||
nonzero_eval = {
|
||||
name: value for name, value in nonzero.items() if name.endswith("-eval.exit_code")
|
||||
}
|
||||
nonzero_infer = {
|
||||
name: value for name, value in nonzero.items() if name.endswith(".infer.exit_code")
|
||||
}
|
||||
# Any other nonzero exit file (nonstandard/unexpected name) must still gate validity;
|
||||
# silently ignoring it could let a broken run be finalized as adopted/rejected.
|
||||
nonzero_other = {
|
||||
name: value
|
||||
for name, value in nonzero.items()
|
||||
if name not in nonzero_eval and name not in nonzero_infer
|
||||
}
|
||||
total = int(eval_metrics.get("total_instances") or 0)
|
||||
resolved = int(eval_metrics.get("resolved_instances") or 0)
|
||||
empty = int(eval_metrics.get("empty_patch_instances") or 0)
|
||||
return {
|
||||
**eval_metrics,
|
||||
"eval_report_count": int(eval_metrics.get("report_count") or 0),
|
||||
"env_delivery_error_count": len(
|
||||
artifacts.get("env_delivery", {}).get("errors", [])
|
||||
),
|
||||
"resolved_rate": (resolved / total) if total else None,
|
||||
"empty_rate": (empty / total) if total else None,
|
||||
"nonzero_eval_exit_codes": nonzero_eval,
|
||||
"nonzero_infer_exit_codes": nonzero_infer,
|
||||
"nonzero_other_exit_codes": nonzero_other,
|
||||
}
|
||||
|
||||
|
||||
def render_analysis(
|
||||
manifest: dict[str, Any],
|
||||
artifacts: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
) -> str:
|
||||
resolved = f"{metrics.get('resolved_instances', 0)}/{metrics.get('total_instances', 0)}"
|
||||
env_delivery = artifacts.get("env_delivery") or {}
|
||||
env_lines: list[str] = []
|
||||
if env_delivery.get("expected"):
|
||||
env_lines = [
|
||||
"- Runtime env delivery checked: "
|
||||
f"{env_delivery.get('checked_instance_count', 0)} metadata files",
|
||||
f"- Runtime env delivery errors: {len(env_delivery.get('errors', []))}",
|
||||
]
|
||||
for error in env_delivery.get("errors", []):
|
||||
env_lines.append(f" - {error}")
|
||||
return "\n".join(
|
||||
[
|
||||
f"# SWE Experiment Analysis: {manifest.get('exp_id')}",
|
||||
"",
|
||||
f"- Question: {manifest.get('question', '')}",
|
||||
f"- Batch: `{artifacts.get('batch_dir', '')}`",
|
||||
f"- Source HEAD: `{manifest.get('source', {}).get('head', '')}`",
|
||||
f"- Handoff HEAD: `{manifest.get('handoff', {}).get('head', '')}`",
|
||||
f"- Eval reports: {metrics.get('eval_report_count', 0)}",
|
||||
f"- Resolved: {resolved}",
|
||||
f"- Empty patches: {metrics.get('empty_patch_instances', 0)}",
|
||||
f"- Errors: {metrics.get('error_instances', 0)}",
|
||||
*env_lines,
|
||||
"",
|
||||
"This report is generated from the manifest and batch artifacts; "
|
||||
"raw traces remain in place.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def render_decision(decision: dict[str, Any], metrics: dict[str, Any]) -> str:
|
||||
resolved = f"{metrics.get('resolved_instances', 0)}/{metrics.get('total_instances', 0)}"
|
||||
return "\n".join(
|
||||
[
|
||||
f"# Decision: {decision['decision']}",
|
||||
"",
|
||||
f"- Experiment: `{decision['exp_id']}`",
|
||||
f"- Reason: {decision['reason']}",
|
||||
f"- Resolved: {resolved}",
|
||||
f"- Empty patches: {metrics.get('empty_patch_instances', 0)}",
|
||||
f"- Decided at: {decision['decided_at']}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def update_current(
|
||||
ledger_root: Path,
|
||||
exp_id: str,
|
||||
decision: str,
|
||||
metrics: dict[str, Any],
|
||||
updated_at: str,
|
||||
) -> None:
|
||||
current = read_json(ledger_root / "current.json")
|
||||
if current.get("active_experiment") == exp_id:
|
||||
current["active_experiment"] = None
|
||||
current.update(
|
||||
{
|
||||
"updated_at": updated_at,
|
||||
"last_experiment": exp_id,
|
||||
"last_decision": decision,
|
||||
"last_result": {
|
||||
"resolved": metrics.get("resolved_instances", 0),
|
||||
"total": metrics.get("total_instances", 0),
|
||||
"empty": metrics.get("empty_patch_instances", 0),
|
||||
},
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
|
||||
|
||||
def update_baseline(
|
||||
ledger_root: Path,
|
||||
model: str,
|
||||
exp_id: str,
|
||||
manifest: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
reason: str,
|
||||
) -> None:
|
||||
baselines = read_json(ledger_root / "baselines.json")
|
||||
baselines[model] = {
|
||||
"current_best": {
|
||||
"exp_id": exp_id,
|
||||
"label": manifest.get("config", {}).get("condition_label", exp_id),
|
||||
"source_head": manifest.get("source", {}).get("head", ""),
|
||||
"resolved": metrics.get("resolved_instances", 0),
|
||||
"total": metrics.get("total_instances", 0),
|
||||
"empty": metrics.get("empty_patch_instances", 0),
|
||||
"reason": reason,
|
||||
}
|
||||
}
|
||||
atomic_write_json(ledger_root / "baselines.json", baselines)
|
||||
|
||||
|
||||
def update_mechanism(
|
||||
ledger_root: Path,
|
||||
mechanism: str,
|
||||
decision: str,
|
||||
exp_id: str,
|
||||
reason: str,
|
||||
) -> None:
|
||||
mechanisms = read_json(ledger_root / "mechanisms.json")
|
||||
mechanisms[mechanism] = {
|
||||
"status": decision,
|
||||
"exp_id": exp_id,
|
||||
"reason": reason,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
atomic_write_json(ledger_root / "mechanisms.json", mechanisms)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a reproducible OpenSquilla experiment manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
DEFAULT_REQUIRED_SECRET_ENV,
|
||||
RUNNER_RELATIVE_PATH,
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
atomic_write_text,
|
||||
copy_snapshot,
|
||||
env_exports_for_command,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
parse_env_overrides,
|
||||
required_secret_env,
|
||||
sh_quote,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
RUN_MODES = {"qwen_only", "glm_only", "both"}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
parser.add_argument("--question", required=True)
|
||||
parser.add_argument("--hypothesis", default="")
|
||||
parser.add_argument("--condition-label", required=True)
|
||||
parser.add_argument("--run-mode", required=True, choices=sorted(RUN_MODES))
|
||||
parser.add_argument("--source-root", type=Path, required=True)
|
||||
parser.add_argument("--handoff-root", type=Path, required=True)
|
||||
parser.add_argument("--qwen-config", type=Path, required=True)
|
||||
parser.add_argument("--glm-config", type=Path, required=True)
|
||||
parser.add_argument("--ml-instance-file", type=Path, required=True)
|
||||
parser.add_argument("--verified-instance-file", type=Path, required=True)
|
||||
parser.add_argument("--ml-count", type=int, required=True)
|
||||
parser.add_argument("--verified-count", type=int, required=True)
|
||||
parser.add_argument("--qwen-workers", type=int, required=True)
|
||||
parser.add_argument("--glm-workers", type=int, required=True)
|
||||
parser.add_argument("--eval-workers", type=int, required=True)
|
||||
parser.add_argument("--env", action="append", default=[])
|
||||
parser.add_argument(
|
||||
"--required-secret-env",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="MODEL=ENV_VAR",
|
||||
help="Override the required provider secret env var for a model (qwen=..., glm=...).",
|
||||
)
|
||||
parser.add_argument("--decision-gate", action="append", default=[])
|
||||
parser.add_argument("--allow-handoff-dirty", action="store_true")
|
||||
parser.add_argument("--resume-existing", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
create_experiment(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def create_experiment(args: argparse.Namespace) -> None:
|
||||
exp_id = validate_exp_id(args.exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
with ledger_lock(ledger_root):
|
||||
if run_dir.exists() and not args.resume_existing and not args.dry_run:
|
||||
raise LedgerError(f"experiment already exists: {run_dir}")
|
||||
|
||||
source = git_info(args.source_root)
|
||||
handoff = git_info(args.handoff_root)
|
||||
if source.dirty_count:
|
||||
raise LedgerError("source repo is dirty; refusing to create experiment manifest")
|
||||
if handoff.dirty_count and not args.allow_handoff_dirty:
|
||||
raise LedgerError("handoff repo is dirty; pass --allow-handoff-dirty to record it")
|
||||
|
||||
qwen_config = _require_config_file(args.qwen_config, "qwen")
|
||||
glm_config = _require_config_file(args.glm_config, "glm")
|
||||
ml_file = _require_file(args.ml_instance_file, "ml instance file")
|
||||
verified_file = _require_file(args.verified_instance_file, "verified instance file")
|
||||
runner = args.handoff_root / RUNNER_RELATIVE_PATH
|
||||
_require_file(runner, "handoff runner")
|
||||
|
||||
config_snapshot_dir = run_dir / "config_snapshot"
|
||||
instance_snapshot_dir = run_dir / "instance_snapshot"
|
||||
if args.dry_run:
|
||||
qwen_snapshot = _snapshot_preview(qwen_config)
|
||||
glm_snapshot = _snapshot_preview(glm_config)
|
||||
ml_snapshot = _snapshot_preview(ml_file)
|
||||
verified_snapshot = _snapshot_preview(verified_file)
|
||||
else:
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
qwen_snapshot = copy_snapshot(qwen_config, config_snapshot_dir / "qwen")
|
||||
glm_snapshot = copy_snapshot(glm_config, config_snapshot_dir / "glm")
|
||||
ml_snapshot = copy_snapshot(ml_file, instance_snapshot_dir / "ml")
|
||||
verified_snapshot = copy_snapshot(verified_file, instance_snapshot_dir / "verified")
|
||||
|
||||
env = required_secret_env(
|
||||
args.run_mode, _required_secret_env_overrides(args.required_secret_env)
|
||||
)
|
||||
env.update(parse_env_overrides(args.env))
|
||||
created_at = now_iso()
|
||||
manifest = {
|
||||
"exp_id": exp_id,
|
||||
"status": "planned",
|
||||
"question": args.question,
|
||||
"hypothesis": args.hypothesis,
|
||||
"source": source.__dict__,
|
||||
"handoff": {
|
||||
**handoff.__dict__,
|
||||
"dirty_allowed": bool(args.allow_handoff_dirty),
|
||||
},
|
||||
"model": _model_metadata(args.run_mode, env),
|
||||
"config": {
|
||||
"condition_label": args.condition_label,
|
||||
"qwen_config": qwen_snapshot,
|
||||
"glm_config": glm_snapshot,
|
||||
"runner": {"path": str(runner), "sha256": sha256_file(runner)},
|
||||
"env": env,
|
||||
},
|
||||
"slice": {
|
||||
"ml": {**ml_snapshot, "count": args.ml_count},
|
||||
"verified": {**verified_snapshot, "count": args.verified_count},
|
||||
},
|
||||
"execution": {
|
||||
"run_mode": args.run_mode,
|
||||
"qwen_workers": args.qwen_workers,
|
||||
"glm_workers": args.glm_workers,
|
||||
"eval_workers": args.eval_workers,
|
||||
"command_path": str(run_dir / "command.sh"),
|
||||
},
|
||||
"artifacts": {},
|
||||
"decision_gate": {"items": args.decision_gate},
|
||||
"created_at": created_at,
|
||||
"evidence_level": "manifested",
|
||||
}
|
||||
|
||||
command = render_command(args, manifest)
|
||||
if not args.dry_run:
|
||||
atomic_write_json(run_dir / "manifest.json", manifest)
|
||||
atomic_write_text(run_dir / "command.sh", command)
|
||||
_make_executable(run_dir / "command.sh")
|
||||
atomic_write_json(
|
||||
run_dir / "preflight.json",
|
||||
{
|
||||
"created_at": created_at,
|
||||
"source_dirty_count": source.dirty_count,
|
||||
"handoff_dirty_count": handoff.dirty_count,
|
||||
"config_hashes": {
|
||||
"qwen": qwen_snapshot["sha256"],
|
||||
"glm": glm_snapshot["sha256"],
|
||||
},
|
||||
},
|
||||
)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": created_at,
|
||||
"exp_id": exp_id,
|
||||
"event": "created",
|
||||
"run_dir": str(run_dir),
|
||||
"condition_label": args.condition_label,
|
||||
},
|
||||
)
|
||||
print(json.dumps({"exp_id": exp_id, "run_dir": str(run_dir)}, indent=2))
|
||||
|
||||
|
||||
def _required_secret_env_overrides(items: list[str]) -> dict[str, str]:
|
||||
mapping = dict(DEFAULT_REQUIRED_SECRET_ENV)
|
||||
for item in items:
|
||||
model, sep, name = item.partition("=")
|
||||
model = model.strip()
|
||||
name = name.strip()
|
||||
if not sep or model not in DEFAULT_REQUIRED_SECRET_ENV or not name:
|
||||
raise LedgerError(
|
||||
"--required-secret-env must use MODEL=ENV_VAR with MODEL in "
|
||||
+ "/".join(sorted(DEFAULT_REQUIRED_SECRET_ENV))
|
||||
)
|
||||
mapping[model] = name
|
||||
return mapping
|
||||
|
||||
|
||||
def _require_config_file(path: Path, label: str) -> Path:
|
||||
if path.is_dir():
|
||||
raise LedgerError(f"{label} config must be a config.toml file, got directory: {path}")
|
||||
return _require_file(path, f"{label} config")
|
||||
|
||||
|
||||
def _require_file(path: Path, label: str) -> Path:
|
||||
if not path.is_file():
|
||||
raise LedgerError(f"missing {label}: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _snapshot_preview(path: Path) -> dict[str, str]:
|
||||
return {"source": str(path), "snapshot": "", "sha256": sha256_file(path)}
|
||||
|
||||
|
||||
def _model_metadata(run_mode: str, env: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
# Thinking levels reflect the pinned env treatment when present; the
|
||||
# defaults mirror what an unpinned run actually gets (the batch runner's
|
||||
# GLM_THINKING:-xhigh fallback, the qwen config.toml thinking_level).
|
||||
return {
|
||||
"run_mode": run_mode,
|
||||
"qwen": {
|
||||
"enabled": run_mode != "glm_only",
|
||||
"provider": "dashscope",
|
||||
"model": "qwen3.6-flash",
|
||||
"thinking": _pinned_env_value(env, "QWEN_THINKING", "high"),
|
||||
"cache": "on",
|
||||
},
|
||||
"glm": {
|
||||
"enabled": run_mode != "qwen_only",
|
||||
"provider": "openrouter",
|
||||
"model": "z-ai/glm-5.1",
|
||||
"thinking": _pinned_env_value(env, "GLM_THINKING", "xhigh"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _pinned_env_value(env: dict[str, dict[str, Any]], key: str, default: str) -> str:
|
||||
meta = env.get(key)
|
||||
if not isinstance(meta, dict) or meta.get("redacted"):
|
||||
return default
|
||||
value = meta.get("value")
|
||||
return value if value else default
|
||||
|
||||
|
||||
def render_command(args: argparse.Namespace, manifest: dict[str, Any]) -> str:
|
||||
qwen_config_dir = Path(_snapshot_or_source(manifest["config"]["qwen_config"])).parent
|
||||
glm_config_dir = Path(_snapshot_or_source(manifest["config"]["glm_config"])).parent
|
||||
ml_instance_file = _snapshot_or_source(manifest["slice"]["ml"])
|
||||
verified_instance_file = _snapshot_or_source(manifest["slice"]["verified"])
|
||||
env_exports = [
|
||||
f"export OPENSQUILLA_SOURCE_REPO={sh_quote(str(args.source_root))}",
|
||||
f"export RUN_MODE={sh_quote(args.run_mode)}",
|
||||
f"export CONDITION_LABEL={sh_quote(args.condition_label)}",
|
||||
f"export QWEN_CONFIG_DIR={sh_quote(str(qwen_config_dir))}",
|
||||
f"export GLM_CONFIG_DIR={sh_quote(str(glm_config_dir))}",
|
||||
f"export ML_INSTANCE_FILE={sh_quote(ml_instance_file)}",
|
||||
f"export VERIFIED_INSTANCE_FILE={sh_quote(verified_instance_file)}",
|
||||
f"export ML_COUNT={args.ml_count}",
|
||||
f"export VERIFIED_COUNT={args.verified_count}",
|
||||
f"export QWEN_WORKERS={args.qwen_workers}",
|
||||
f"export GLM_WORKERS={args.glm_workers}",
|
||||
f"export EVAL_WORKERS={args.eval_workers}",
|
||||
]
|
||||
env_exports.extend(env_exports_for_command(manifest["config"]["env"]))
|
||||
return "\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
"# Secrets are intentionally not embedded. Provide provider keys via",
|
||||
"# environment variables or stdin as expected by the handoff runner.",
|
||||
f"cd {sh_quote(str(args.handoff_root))}",
|
||||
*env_exports,
|
||||
f"{sh_quote(str(args.handoff_root / RUNNER_RELATIVE_PATH))}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_or_source(payload: dict[str, Any]) -> str:
|
||||
snapshot = str(payload.get("snapshot") or "")
|
||||
return snapshot or str(payload["source"])
|
||||
|
||||
|
||||
def _make_executable(path: Path) -> None:
|
||||
mode = path.stat().st_mode
|
||||
path.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Register and query quarantined (contaminated) runs in the experiment ledger.
|
||||
|
||||
A contamination class tags artifact batches whose results are confounded by an
|
||||
infra defect (e.g. a provider-compaction marker leak). Registration is
|
||||
idempotent: names merge into ``contaminations.json`` at the ledger root,
|
||||
matching ledger run dirs get a ``contamination.json`` stamp, and an event is
|
||||
appended to ``experiments.jsonl``. Baseline tooling and ``exp_status`` read the
|
||||
registry to warn when a quarantined artifact backs a baseline.
|
||||
|
||||
Usage:
|
||||
exp_quarantine.py register --contamination-class NAME --names-file F \
|
||||
--description TEXT [--evidence PATH] [--boundary-commit C ...]
|
||||
exp_quarantine.py check ARTIFACT [ARTIFACT ...]
|
||||
exp_quarantine.py list
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
artifact_basename,
|
||||
atomic_write_json,
|
||||
contamination_class_for,
|
||||
contaminations_path,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
load_contaminations,
|
||||
now_iso,
|
||||
read_json,
|
||||
)
|
||||
|
||||
CLASS_NAME_RE_HELP = "lowercase snake_case, e.g. tool_result_compaction_defect"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
register = sub.add_parser("register", help="register/merge a contamination class")
|
||||
register.add_argument(
|
||||
"--contamination-class",
|
||||
required=True,
|
||||
help=f"class name ({CLASS_NAME_RE_HELP})",
|
||||
)
|
||||
register.add_argument(
|
||||
"--names-file",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="JSON file containing a list of artifact batch names",
|
||||
)
|
||||
register.add_argument("--description", required=True)
|
||||
register.add_argument(
|
||||
"--evidence",
|
||||
default="",
|
||||
help="path or URL of the audit/report that established the contamination",
|
||||
)
|
||||
register.add_argument(
|
||||
"--boundary-commit",
|
||||
action="append",
|
||||
default=[],
|
||||
help="fix-boundary commit (repeatable); runs at or before these are affected",
|
||||
)
|
||||
|
||||
check = sub.add_parser("check", help="check artifact names/paths against the registry")
|
||||
check.add_argument("artifacts", nargs="+")
|
||||
|
||||
sub.add_parser("list", help="summarize registered contamination classes")
|
||||
return parser
|
||||
|
||||
|
||||
def _load_names(path: Path) -> list[str]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise LedgerError(f"cannot read names file {path}: {exc}") from exc
|
||||
if not isinstance(data, list) or not all(isinstance(item, str) for item in data):
|
||||
raise LedgerError(f"names file must be a JSON list of strings: {path}")
|
||||
# Normalize before filtering: a raw value like "/" or "///" is non-empty
|
||||
# pre-normalization but collapses to "" via artifact_basename, and an
|
||||
# empty name would substring-match (or exact-match) every run.
|
||||
names = sorted(
|
||||
{
|
||||
normalized
|
||||
for item in data
|
||||
if item.strip()
|
||||
for normalized in [artifact_basename(item)]
|
||||
if normalized
|
||||
}
|
||||
)
|
||||
if not names:
|
||||
raise LedgerError(f"names file is empty: {path}")
|
||||
return names
|
||||
|
||||
|
||||
def _candidate_names_in_payload(payload: Any) -> set[str]:
|
||||
"""Recursively collect basename-normalized string values from a JSON payload."""
|
||||
names: set[str] = set()
|
||||
if isinstance(payload, str):
|
||||
normalized = artifact_basename(payload)
|
||||
if normalized:
|
||||
names.add(normalized)
|
||||
elif isinstance(payload, dict):
|
||||
for value in payload.values():
|
||||
names.update(_candidate_names_in_payload(value))
|
||||
elif isinstance(payload, list):
|
||||
for item in payload:
|
||||
names.update(_candidate_names_in_payload(item))
|
||||
return names
|
||||
|
||||
|
||||
def _stamp_run_dirs(root: Path, class_name: str, names: list[str]) -> list[str]:
|
||||
"""Stamp ledger run dirs whose artifacts reference a quarantined batch name.
|
||||
|
||||
Matching is by exact basename, consistent with ``contamination_class_for``.
|
||||
A prior substring-based check over the raw JSON text could both false-
|
||||
positive (a quarantined name that is a substring of an unrelated, longer
|
||||
name) and, worse, treat an empty name as a substring of everything.
|
||||
"""
|
||||
stamped: list[str] = []
|
||||
runs_dir = root / "runs"
|
||||
if not runs_dir.is_dir():
|
||||
return stamped
|
||||
name_set = set(names)
|
||||
for run_dir in sorted(runs_dir.iterdir()):
|
||||
if not run_dir.is_dir():
|
||||
continue
|
||||
matched: set[str] = set()
|
||||
for source in ("artifacts.json", "manifest.json"):
|
||||
payload = read_json(run_dir / source)
|
||||
if not payload:
|
||||
continue
|
||||
matched.update(name_set & _candidate_names_in_payload(payload))
|
||||
if not matched:
|
||||
continue
|
||||
stamp_path = run_dir / "contamination.json"
|
||||
existing = read_json(stamp_path)
|
||||
classes = existing.get("classes") if isinstance(existing.get("classes"), dict) else {}
|
||||
previous = classes.get(class_name, {})
|
||||
previous_names = (
|
||||
set(previous.get("matched_artifact_names", []))
|
||||
if isinstance(previous, dict)
|
||||
else set()
|
||||
)
|
||||
classes[class_name] = {
|
||||
"matched_artifact_names": sorted(previous_names | matched),
|
||||
"stamped_at": (
|
||||
previous.get("stamped_at")
|
||||
if isinstance(previous, dict) and previous.get("stamped_at")
|
||||
else now_iso()
|
||||
),
|
||||
}
|
||||
atomic_write_json(stamp_path, {"classes": classes})
|
||||
stamped.append(run_dir.name)
|
||||
return stamped
|
||||
|
||||
|
||||
def cmd_register(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
class_name = str(args.contamination_class).strip()
|
||||
if not class_name:
|
||||
raise LedgerError("--contamination-class must be non-empty")
|
||||
names = _load_names(args.names_file)
|
||||
with ledger_lock(root):
|
||||
data = load_contaminations(root)
|
||||
classes = data["classes"]
|
||||
entry = classes.get(class_name)
|
||||
if not isinstance(entry, dict):
|
||||
entry = {"registered_at": now_iso()}
|
||||
existing_names = set(entry.get("artifact_names", []))
|
||||
merged = sorted(existing_names | set(names))
|
||||
new_names = sorted(set(names) - existing_names)
|
||||
entry.update(
|
||||
{
|
||||
"description": str(args.description),
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"artifact_names": merged,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
)
|
||||
classes[class_name] = entry
|
||||
data["updated_at"] = now_iso()
|
||||
atomic_write_json(contaminations_path(root), data)
|
||||
stamped = _stamp_run_dirs(root, class_name, merged)
|
||||
append_jsonl(
|
||||
root / "experiments.jsonl",
|
||||
{
|
||||
"event": "contamination_registered",
|
||||
"contamination_class": class_name,
|
||||
"artifact_names_total": len(merged),
|
||||
"artifact_names_new": len(new_names),
|
||||
"stamped_run_dirs": stamped,
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"time": now_iso(),
|
||||
},
|
||||
)
|
||||
print(
|
||||
f"registered {class_name}: {len(merged)} artifact names "
|
||||
f"({len(new_names)} new), stamped {len(stamped)} ledger run dirs"
|
||||
)
|
||||
for run_name in stamped:
|
||||
print(f" stamped: runs/{run_name}/contamination.json")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
contaminations = load_contaminations(root)
|
||||
dirty = 0
|
||||
for artifact in args.artifacts:
|
||||
contamination_class = contamination_class_for(root, artifact, contaminations)
|
||||
if contamination_class:
|
||||
dirty += 1
|
||||
print(f"QUARANTINED\t{artifact_basename(artifact)}\t{contamination_class}")
|
||||
else:
|
||||
print(f"clean\t{artifact_basename(artifact)}")
|
||||
return 1 if dirty else 0
|
||||
|
||||
|
||||
def cmd_list(_: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
classes = load_contaminations(root).get("classes", {})
|
||||
if not classes:
|
||||
print("no contamination classes registered")
|
||||
return 0
|
||||
for name, payload in sorted(classes.items()):
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
count = len(payload.get("artifact_names", []))
|
||||
boundary = ",".join(payload.get("boundary_commits", [])) or "-"
|
||||
print(f"{name}\truns={count}\tboundary={boundary}")
|
||||
description = payload.get("description")
|
||||
if description:
|
||||
print(f" {description}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "register":
|
||||
return cmd_register(args)
|
||||
if args.command == "check":
|
||||
return cmd_check(args)
|
||||
return cmd_list(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run an OpenSquilla experiment from an existing ledger manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
read_json,
|
||||
read_json_strict,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
return run_experiment(args.exp_id)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def run_experiment(exp_id: str) -> int:
|
||||
exp_id = validate_exp_id(exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
manifest_path = run_dir / "manifest.json"
|
||||
manifest = read_json_strict(manifest_path, label="experiment manifest")
|
||||
_verify_manifest_inputs(manifest)
|
||||
command_path = Path(manifest["execution"]["command_path"])
|
||||
if not command_path.is_file():
|
||||
raise LedgerError(f"missing command.sh: {command_path}")
|
||||
|
||||
started_at = now_iso()
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(
|
||||
run_dir / "live_status.json",
|
||||
{"status": "running", "started_at": started_at, "exp_id": exp_id},
|
||||
)
|
||||
current = read_json(ledger_root / "current.json")
|
||||
current.update(
|
||||
{
|
||||
"updated_at": started_at,
|
||||
"active_experiment": exp_id,
|
||||
"active_run_dir": str(run_dir),
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{"time": started_at, "exp_id": exp_id, "event": "started", "run_dir": str(run_dir)},
|
||||
)
|
||||
|
||||
status = "finished"
|
||||
return_code = 0
|
||||
failure = ""
|
||||
try:
|
||||
proc = subprocess.run([str(command_path)], cwd=run_dir, check=False)
|
||||
return_code = proc.returncode
|
||||
except KeyboardInterrupt:
|
||||
status = "interrupted"
|
||||
return_code = 130
|
||||
failure = "keyboard_interrupt"
|
||||
except Exception as exc: # pragma: no cover - defensive ledger cleanup path
|
||||
status = "failed"
|
||||
return_code = 2
|
||||
failure = str(exc)
|
||||
finally:
|
||||
_record_completion(
|
||||
ledger_root=ledger_root,
|
||||
run_dir=run_dir,
|
||||
exp_id=exp_id,
|
||||
started_at=started_at,
|
||||
status=status,
|
||||
return_code=return_code,
|
||||
failure=failure,
|
||||
)
|
||||
return return_code
|
||||
|
||||
|
||||
def _record_completion(
|
||||
*,
|
||||
ledger_root: Path,
|
||||
run_dir: Path,
|
||||
exp_id: str,
|
||||
started_at: str,
|
||||
status: str,
|
||||
return_code: int,
|
||||
failure: str,
|
||||
) -> None:
|
||||
finished_at = now_iso()
|
||||
payload: dict[str, Any] = {
|
||||
"status": status,
|
||||
"started_at": started_at,
|
||||
"finished_at": finished_at,
|
||||
"return_code": return_code,
|
||||
"exp_id": exp_id,
|
||||
}
|
||||
if failure:
|
||||
payload["failure"] = failure
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(run_dir / "live_status.json", payload)
|
||||
current = read_json(ledger_root / "current.json")
|
||||
if current.get("active_experiment") == exp_id:
|
||||
current["active_experiment"] = None
|
||||
current.update(
|
||||
{
|
||||
"updated_at": finished_at,
|
||||
"last_experiment": exp_id,
|
||||
"last_return_code": return_code,
|
||||
"last_status": status,
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": finished_at,
|
||||
"exp_id": exp_id,
|
||||
"event": status,
|
||||
"return_code": return_code,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _verify_manifest_inputs(manifest: dict[str, Any]) -> None:
|
||||
source = manifest.get("source", {})
|
||||
current_source = git_info(Path(source["path"]))
|
||||
if current_source.dirty_count:
|
||||
raise LedgerError("source repo is dirty; refusing to run experiment")
|
||||
if current_source.head != source.get("head"):
|
||||
raise LedgerError("source HEAD changed since manifest creation")
|
||||
for section in ("qwen_config", "glm_config"):
|
||||
payload = manifest.get("config", {}).get(section, {})
|
||||
_verify_payload_hash(section, payload)
|
||||
for section in ("ml", "verified"):
|
||||
payload = manifest.get("slice", {}).get(section, {})
|
||||
_verify_payload_hash(f"{section} instance file", payload)
|
||||
_verify_runner(manifest.get("config", {}).get("runner", {}))
|
||||
|
||||
|
||||
def _verify_runner(runner: dict[str, Any]) -> None:
|
||||
expected = str(runner.get("sha256") or "")
|
||||
path_str = str(runner.get("path") or "")
|
||||
if not expected or not path_str:
|
||||
return
|
||||
path = Path(path_str)
|
||||
if not path.is_file():
|
||||
raise LedgerError(f"handoff runner missing: {path}")
|
||||
if sha256_file(path) != expected:
|
||||
raise LedgerError("handoff runner changed since manifest creation")
|
||||
|
||||
|
||||
def _verify_payload_hash(label: str, payload: dict[str, Any]) -> None:
|
||||
expected = payload.get("sha256")
|
||||
if not expected:
|
||||
return
|
||||
checked = False
|
||||
for key in ("snapshot", "source"):
|
||||
path_str = str(payload.get(key) or "")
|
||||
if not path_str:
|
||||
continue
|
||||
path = Path(path_str)
|
||||
if path.is_file():
|
||||
checked = True
|
||||
if sha256_file(path) != expected:
|
||||
raise LedgerError(f"{label} {key} hash changed since manifest creation")
|
||||
if not checked:
|
||||
raise LedgerError(f"{label} source and snapshot are both missing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Print the current OpenSquilla experiment ledger status."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
active_processes,
|
||||
active_swe_containers,
|
||||
contamination_class_for,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_root_from_env,
|
||||
load_contaminations,
|
||||
read_json,
|
||||
run_dir_contamination_classes,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-root", type=Path, required=True)
|
||||
parser.add_argument("--handoff-root", type=Path, required=True)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
status = collect_status(args.source_root, args.handoff_root)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
if args.json:
|
||||
print(json.dumps(status, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(render_text(status))
|
||||
return 0
|
||||
|
||||
|
||||
def collect_status(source_root: Path, handoff_root: Path) -> dict[str, Any]:
|
||||
ledger_root = ledger_root_from_env()
|
||||
current = read_json(ledger_root / "current.json")
|
||||
baselines = read_json(ledger_root / "baselines.json")
|
||||
mechanisms = read_json(ledger_root / "mechanisms.json")
|
||||
source = git_info(source_root)
|
||||
handoff = git_info(handoff_root)
|
||||
processes = active_processes()
|
||||
warnings = []
|
||||
active_exp_id = current.get("active_experiment")
|
||||
active_manifest = None
|
||||
active_manifest_payload: dict[str, Any] = {}
|
||||
active_live_status: dict[str, Any] | None = None
|
||||
if active_exp_id:
|
||||
run_dir = exp_dir(ledger_root, str(active_exp_id))
|
||||
manifest_path = run_dir / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
active_manifest = str(manifest_path)
|
||||
active_manifest_payload = read_json(manifest_path)
|
||||
else:
|
||||
warnings.append(f"active experiment manifest missing: {active_exp_id}")
|
||||
live_status_path = run_dir / "live_status.json"
|
||||
if live_status_path.exists():
|
||||
active_live_status = read_json(live_status_path)
|
||||
containers = active_swe_containers(active_manifest_payload)
|
||||
if active_exp_id and not processes and not containers:
|
||||
warnings.append(
|
||||
f"active experiment {active_exp_id} but no live runner processes or SWE "
|
||||
"containers; verify live_status.json (stale active state?)"
|
||||
)
|
||||
qwen_baseline = _extract_baseline(baselines, "qwen")
|
||||
glm_baseline = _extract_baseline(baselines, "glm")
|
||||
for label, baseline in (("qwen", qwen_baseline), ("glm", glm_baseline)):
|
||||
head = baseline.get("source_head")
|
||||
if (
|
||||
head
|
||||
and not str(source.head).startswith(str(head))
|
||||
and not str(head).startswith(source.short_head)
|
||||
):
|
||||
warnings.append(f"{label} baseline source_head differs from current source HEAD")
|
||||
contaminations = load_contaminations(ledger_root)
|
||||
for label, baseline in (("qwen", qwen_baseline), ("glm", glm_baseline)):
|
||||
contamination_classes: list[str] = []
|
||||
artifact = baseline.get("artifact")
|
||||
if artifact:
|
||||
artifact_class = contamination_class_for(
|
||||
ledger_root, str(artifact), contaminations
|
||||
)
|
||||
if artifact_class:
|
||||
contamination_classes.append(artifact_class)
|
||||
baseline_exp_id = baseline.get("exp_id")
|
||||
if baseline_exp_id:
|
||||
contamination_classes.extend(
|
||||
run_dir_contamination_classes(ledger_root, str(baseline_exp_id))
|
||||
)
|
||||
if contamination_classes:
|
||||
joined = ", ".join(sorted(set(contamination_classes)))
|
||||
warnings.append(
|
||||
f"{label} baseline is quarantined ({joined}); "
|
||||
"re-baseline on clean runs"
|
||||
)
|
||||
return {
|
||||
"ledger_root": str(ledger_root),
|
||||
"source": source.__dict__,
|
||||
"handoff": handoff.__dict__,
|
||||
"active_experiment": active_exp_id,
|
||||
"active_manifest": active_manifest,
|
||||
"active_live_status": active_live_status,
|
||||
"processes": processes,
|
||||
"containers": containers,
|
||||
"qwen_baseline": qwen_baseline,
|
||||
"glm_baseline": glm_baseline,
|
||||
"mechanisms": mechanisms,
|
||||
"warnings": [*current.get("warnings", []), *warnings]
|
||||
if isinstance(current.get("warnings", []), list)
|
||||
else warnings,
|
||||
"current": current,
|
||||
}
|
||||
|
||||
|
||||
def _extract_baseline(baselines: dict[str, Any], model: str) -> dict[str, Any]:
|
||||
value = baselines.get(model, {})
|
||||
if isinstance(value, dict) and isinstance(value.get("current_best"), dict):
|
||||
return dict(value["current_best"])
|
||||
if isinstance(value, dict) and isinstance(value.get("latest_guard"), dict):
|
||||
return dict(value["latest_guard"])
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {}
|
||||
|
||||
|
||||
def render_text(status: dict[str, Any]) -> str:
|
||||
source = status["source"]
|
||||
handoff = status["handoff"]
|
||||
lines = [
|
||||
f"Ledger: {status['ledger_root']}",
|
||||
f"Source: {_dirty_label(source)} {source['short_head']} {source['branch']}",
|
||||
f"Handoff: {_dirty_label(handoff)} {handoff['short_head']} {handoff['branch']}",
|
||||
]
|
||||
active = status.get("active_experiment")
|
||||
if active:
|
||||
lines.append(f"Active experiment: {active}")
|
||||
if status.get("active_manifest"):
|
||||
lines.append(f"Manifest: {status['active_manifest']}")
|
||||
live = status.get("active_live_status") or {}
|
||||
if live:
|
||||
lines.append(f"Live status: {live.get('status', 'unknown')}")
|
||||
else:
|
||||
lines.append("Active experiment: none")
|
||||
lines.append(f"Active runner processes: {len(status['processes'])}")
|
||||
lines.append(f"Active SWE containers: {len(status['containers'])}")
|
||||
qwen = status.get("qwen_baseline") or {}
|
||||
glm = status.get("glm_baseline") or {}
|
||||
lines.append(f"Qwen baseline: {format_baseline(qwen)}")
|
||||
lines.append(f"GLM baseline: {format_baseline(glm)}")
|
||||
rejected = _mechanisms_by_status(
|
||||
status.get("mechanisms", {}),
|
||||
{"rejected", "rejected_for_qwen"},
|
||||
)
|
||||
observe = _mechanisms_by_status(status.get("mechanisms", {}), {"observe"})
|
||||
if rejected:
|
||||
lines.append("Rejected: " + ", ".join(rejected[:8]))
|
||||
if observe:
|
||||
lines.append("Observe-only: " + ", ".join(observe[:8]))
|
||||
warnings = status.get("warnings") or []
|
||||
if warnings:
|
||||
lines.append("Warnings:")
|
||||
lines.extend(f" - {item}" for item in warnings)
|
||||
lines.append("Next: create or run from manifest; do not infer config from chat.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _dirty_label(info: dict[str, Any]) -> str:
|
||||
if int(info.get("dirty_count") or 0) == 0:
|
||||
return "clean"
|
||||
return f"dirty {info['dirty_count']}"
|
||||
|
||||
|
||||
def format_baseline(baseline: dict[str, Any]) -> str:
|
||||
if not baseline:
|
||||
return "not recorded"
|
||||
result = baseline.get("result")
|
||||
if result:
|
||||
return str(result)
|
||||
resolved = baseline.get("resolved")
|
||||
total = baseline.get("total")
|
||||
empty = baseline.get("empty")
|
||||
label = baseline.get("label") or baseline.get("exp_id") or "baseline"
|
||||
if resolved is not None and total is not None:
|
||||
suffix = f" empty={empty}" if empty is not None else ""
|
||||
return f"{label} = {resolved}/{total}{suffix}"
|
||||
return label
|
||||
|
||||
|
||||
def _mechanisms_by_status(mechanisms: dict[str, Any], statuses: set[str]) -> list[str]:
|
||||
result = []
|
||||
for name, payload in mechanisms.items():
|
||||
if isinstance(payload, dict) and payload.get("status") in statuses:
|
||||
result.append(name)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline transcript replay for the finalize-time red-evidence gate.
|
||||
|
||||
Feeds recorded run transcripts through the exact same pure tracker the
|
||||
live agent loop uses (``opensquilla.engine.finalize_evidence_gate``) and
|
||||
reports whether the gate would have challenged the run's final state. This
|
||||
lets the gate be validated offline: it should fire on runs whose transcripts
|
||||
show red self-evidence at finalization while staying quiet on runs that
|
||||
finished green.
|
||||
|
||||
Usage:
|
||||
# Single runs, one JSON report line per run dir
|
||||
python scripts/experiments/replay_finalize_gate.py RUN_DIR [RUN_DIR ...]
|
||||
|
||||
# Driver mode over a replay-set manifest
|
||||
python scripts/experiments/replay_finalize_gate.py --sets replay_sets.json
|
||||
|
||||
The replay-set manifest maps set names to ``[label, run_dir]`` pairs; the set
|
||||
named ``positives`` is expected to fire, all other sets are controls.
|
||||
|
||||
Each RUN_DIR must contain ``transcript.jsonl`` (persisted session messages)
|
||||
and is expected to contain ``git.patch`` (final workspace diff; missing or
|
||||
blank means no diff, which suppresses the gate exactly as in the live loop).
|
||||
|
||||
Known live-vs-replay divergences:
|
||||
|
||||
- ``has_workspace_diff`` comes from the harness-cleaned ``git.patch``; the
|
||||
live gate uses ``git status --porcelain --untracked-files=all``, which also
|
||||
sees untracked scratch files. Replay therefore under-fires on runs whose
|
||||
only diff was untracked files.
|
||||
- The live gate evaluates only when the model emits a zero-tool-call
|
||||
assistant message (a finalize attempt). Transcripts that end mid-loop
|
||||
(iteration cap, abort) never reach the gate live, so replay reports them
|
||||
as ``should_challenge: false`` with ``suppressed_reason:
|
||||
no_finalize_attempt_at_transcript_end`` (raw triggers stay in the report
|
||||
for diagnostics).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.engine.finalize_evidence_gate import (
|
||||
EXECUTION_TOOL_NAMES,
|
||||
WRITE_TOOL_NAMES,
|
||||
FinalizeEvidenceTracker,
|
||||
execution_signals_from_result,
|
||||
)
|
||||
|
||||
_ANCHOR_MARKERS = (
|
||||
"failed",
|
||||
"failure",
|
||||
"error",
|
||||
"exception",
|
||||
"traceback",
|
||||
"assert",
|
||||
"expected",
|
||||
"actual",
|
||||
)
|
||||
|
||||
|
||||
def _failure_anchor_lines(text: str, limit: int = 3) -> list[str]:
|
||||
"""Lightweight stand-in for the live loop's anchor extraction.
|
||||
|
||||
Anchors only decorate challenge messages and dedup keys; they do not
|
||||
affect whether a trigger fires, so replay uses a simplified extraction.
|
||||
"""
|
||||
|
||||
anchors: list[str] = []
|
||||
for raw_line in text.splitlines():
|
||||
line = " ".join(raw_line.strip().split())
|
||||
if not line:
|
||||
continue
|
||||
lowered = line.lower()
|
||||
if "no failures" in lowered or "no errors" in lowered:
|
||||
continue
|
||||
if not any(marker in lowered for marker in _ANCHOR_MARKERS):
|
||||
continue
|
||||
anchors.append(line[:220])
|
||||
if len(anchors) >= limit:
|
||||
break
|
||||
return anchors
|
||||
|
||||
|
||||
def _string_arg(arguments: dict[str, Any] | None, *names: str) -> str | None:
|
||||
if not isinstance(arguments, dict):
|
||||
return None
|
||||
for name in names:
|
||||
value = arguments.get(name)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _command_for_tool_call(tool_name: str, arguments: dict[str, Any] | None) -> str | None:
|
||||
# Mirrors Agent._execution_command_for_progress.
|
||||
if tool_name == "execute_code":
|
||||
return _string_arg(arguments, "code")
|
||||
return _string_arg(arguments, "command", "cmd")
|
||||
|
||||
|
||||
def _content_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text") or ""))
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def replay_run(run_dir: Path) -> dict[str, Any]:
|
||||
"""Replay one run directory; returns a JSON-safe report."""
|
||||
|
||||
transcript_path = run_dir / "transcript.jsonl"
|
||||
if not transcript_path.is_file():
|
||||
return {"run_dir": str(run_dir), "error": "missing transcript.jsonl"}
|
||||
|
||||
git_patch = run_dir / "git.patch"
|
||||
has_workspace_diff = git_patch.is_file() and bool(
|
||||
git_patch.read_text(errors="replace").strip()
|
||||
)
|
||||
|
||||
tracker = FinalizeEvidenceTracker()
|
||||
pending_calls: dict[str, tuple[str, dict[str, Any] | None]] = {}
|
||||
iteration = 0
|
||||
parse_errors = 0
|
||||
last_assistant_had_tool_calls: bool | None = None
|
||||
|
||||
with transcript_path.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
parse_errors += 1
|
||||
continue
|
||||
message = entry.get("message")
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
role = message.get("role")
|
||||
if role == "assistant":
|
||||
content = message.get("content")
|
||||
saw_tool_call = False
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "toolCall":
|
||||
saw_tool_call = True
|
||||
call_id = str(block.get("id") or "")
|
||||
pending_calls[call_id] = (
|
||||
str(block.get("name") or ""),
|
||||
block.get("arguments")
|
||||
if isinstance(block.get("arguments"), dict)
|
||||
else None,
|
||||
)
|
||||
if saw_tool_call:
|
||||
iteration += 1
|
||||
last_assistant_had_tool_calls = saw_tool_call
|
||||
continue
|
||||
if role != "toolResult":
|
||||
continue
|
||||
tool_name = str(message.get("toolName") or "")
|
||||
call_id = str(message.get("toolCallId") or "")
|
||||
# Look up without popping: an approval retry emits TWO toolResult
|
||||
# messages for the same call id (approval_pending, then the real
|
||||
# retried outcome); the second must still see the arguments.
|
||||
_, arguments = pending_calls.get(call_id, (tool_name, None))
|
||||
is_error = bool(message.get("isError"))
|
||||
if tool_name in WRITE_TOOL_NAMES:
|
||||
tracker.observe_write(
|
||||
_string_arg(arguments, "path", "file_path"),
|
||||
is_error=is_error,
|
||||
iteration=iteration,
|
||||
scratch=(tool_name == "write_scratch"),
|
||||
)
|
||||
elif tool_name in EXECUTION_TOOL_NAMES:
|
||||
command = _command_for_tool_call(tool_name, arguments)
|
||||
if command is None:
|
||||
continue
|
||||
content_text = _content_text(message.get("content"))
|
||||
execution_status = message.get("executionStatus")
|
||||
red, exit_code, timed_out, status_reason = execution_signals_from_result(
|
||||
tool_name=tool_name,
|
||||
content_text=content_text,
|
||||
execution_status=(
|
||||
execution_status if isinstance(execution_status, dict) else None
|
||||
),
|
||||
is_error=is_error,
|
||||
)
|
||||
tracker.observe_execution(
|
||||
command,
|
||||
red=red,
|
||||
exit_code=exit_code,
|
||||
timed_out=timed_out,
|
||||
status_reason=status_reason,
|
||||
failure_anchors=_failure_anchor_lines(content_text) if red else (),
|
||||
iteration=iteration,
|
||||
)
|
||||
|
||||
observation = tracker.build_observation(has_workspace_diff=has_workspace_diff)
|
||||
# Live parity: the gate only runs on a zero-tool-call assistant message.
|
||||
# A transcript that ends mid-loop (iteration cap, abort) never reached it.
|
||||
finalize_attempt_at_end = last_assistant_had_tool_calls is False
|
||||
report = {
|
||||
"run_dir": str(run_dir),
|
||||
"instance": run_dir.name,
|
||||
"iterations": iteration,
|
||||
"parse_errors": parse_errors,
|
||||
"has_workspace_diff": has_workspace_diff,
|
||||
"has_workspace_diff_source": "git.patch",
|
||||
"finalize_attempt_at_end": finalize_attempt_at_end,
|
||||
**observation.to_event_details(),
|
||||
}
|
||||
if not finalize_attempt_at_end:
|
||||
report["should_challenge"] = False
|
||||
report["suppressed_reason"] = "no_finalize_attempt_at_transcript_end"
|
||||
return report
|
||||
|
||||
|
||||
def _print_report(report: dict[str, Any]) -> None:
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
|
||||
|
||||
def _run_sets(sets_path: Path) -> int:
|
||||
manifest = json.loads(sets_path.read_text())
|
||||
summary: dict[str, dict[str, Any]] = {}
|
||||
exit_code = 0
|
||||
for set_name, entries in manifest.items():
|
||||
expected_fire = set_name == "positives"
|
||||
fired: list[str] = []
|
||||
quiet: list[str] = []
|
||||
errored: list[str] = []
|
||||
trigger_counts: dict[str, int] = {}
|
||||
for entry in entries:
|
||||
label, run_dir = entry[0], Path(entry[1])
|
||||
report = replay_run(run_dir)
|
||||
report["set"] = set_name
|
||||
report["label"] = label
|
||||
_print_report(report)
|
||||
if report.get("error"):
|
||||
errored.append(label)
|
||||
continue
|
||||
if report.get("should_challenge"):
|
||||
fired.append(label)
|
||||
for trigger in report.get("triggers") or []:
|
||||
trigger_counts[trigger] = trigger_counts.get(trigger, 0) + 1
|
||||
else:
|
||||
quiet.append(label)
|
||||
summary[set_name] = {
|
||||
"expected_fire": expected_fire,
|
||||
"total": len(entries),
|
||||
"fired": len(fired),
|
||||
"quiet": len(quiet),
|
||||
"errored": len(errored),
|
||||
"fired_labels": fired if expected_fire else fired,
|
||||
"trigger_counts": trigger_counts,
|
||||
}
|
||||
print("\n=== finalize-evidence-gate replay summary ===", file=sys.stderr)
|
||||
for set_name, stats in summary.items():
|
||||
rate = stats["fired"] / stats["total"] if stats["total"] else 0.0
|
||||
print(
|
||||
f"{set_name}: fired {stats['fired']}/{stats['total']} ({rate:.0%})"
|
||||
f" errored={stats['errored']} triggers={stats['trigger_counts']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if stats["expected_fire"] and stats["fired"] != stats["total"]:
|
||||
missing = stats["total"] - stats["fired"] - stats["errored"]
|
||||
print(f" positives not fired: {missing}", file=sys.stderr)
|
||||
return exit_code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("run_dirs", nargs="*", type=Path, help="Run directories to replay")
|
||||
parser.add_argument("--sets", type=Path, help="Replay-set manifest (replay_sets.json)")
|
||||
args = parser.parse_args(argv)
|
||||
if args.sets:
|
||||
return _run_sets(args.sets)
|
||||
if not args.run_dirs:
|
||||
parser.error("provide RUN_DIR arguments or --sets")
|
||||
for run_dir in args.run_dirs:
|
||||
_print_report(replay_run(run_dir))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user