yvgude--lean-ctx/bench/agent-task/swebench_harness/preflight.py

"""Preflight: verify the machine can run the benchmark before spending money.

Checks everything the runbook needs — binaries, auth, docker, network, the
frozen task lock — and prints one PASS/FAIL line each. Exit 1 if anything
required for the requested stage is missing.

Usage:
    python3 -m swebench_harness.preflight             # checks for agent runs
    python3 -m swebench_harness.preflight --evaluate  # also checks docker
"""

from __future__ import annotations

import argparse
import json
import os
import shutil
import subprocess
import sys
import urllib.request

from . import BENCH_ROOT, load_config


def run(cmd: list) -> "tuple[int, str]":
    try:
        res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=30)
        return res.returncode, (res.stdout or "").strip()
    except FileNotFoundError:
        return 127, "not found"
    except subprocess.TimeoutExpired:
        return 124, "timed out"


class Preflight:
    def __init__(self) -> None:
        self.failed = False

    def check(self, name: str, ok: bool, detail: str) -> None:
        print(f"  [{'PASS' if ok else 'FAIL'}] {name:<28} {detail}")
        if not ok:
            self.failed = True


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--evaluate", action="store_true", help="also check the docker evaluation prerequisites")
    args = ap.parse_args()

    cfg = load_config()
    pf = Preflight()
    print("agent-run prerequisites:")

    code, out = run([cfg["agent"]["binary"], "--version"])
    pf.check("claude CLI", code == 0, out.splitlines()[0] if out else "not found")

    code, out = run([cfg["leanctx"]["binary"], "--version"])
    pf.check("lean-ctx CLI", code == 0, out.splitlines()[0] if out else "not found")

    code, out = run(["git", "--version"])
    pf.check("git", code == 0, out)

    has_key = bool(os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN"))
    pf.check(
        "ANTHROPIC_API_KEY",
        has_key,
        "set" if has_key else "missing — fresh-HOME runs have no stored login (export it first)",
    )

    lock = BENCH_ROOT / "tasks.lock.json"
    if lock.exists():
        n = json.loads(lock.read_text())["n"]
        pf.check("tasks.lock.json", True, f"frozen, n={n}")
    else:
        pf.check("tasks.lock.json", False, "missing — python3 -m swebench_harness.select_tasks (one-time)")

    try:
        with urllib.request.urlopen("https://github.com", timeout=10) as resp:
            pf.check("github.com reachable", resp.status < 500, f"HTTP {resp.status} (repo mirrors clone from here)")
    except OSError as e:
        pf.check("github.com reachable", False, str(e))

    free_gb = shutil.disk_usage(BENCH_ROOT).free / 1e9
    pf.check("disk space", free_gb > 20, f"{free_gb:.0f} GB free (mirrors ~2 GB, eval images need more)")

    if args.evaluate:
        print("evaluation prerequisites:")
        code, out = run(["docker", "info", "--format", "{{.ServerVersion}}"])
        pf.check("docker daemon", code == 0, f"server {out}" if code == 0 else out[-120:])
        code, out = run([sys.executable, "-c", "import swebench; print(swebench.__version__)"])
        pf.check("swebench package", code == 0, out if code == 0 else "pip install -r requirements.txt")

    if pf.failed:
        sys.exit(1)
    print("all checks passed.")


if __name__ == "__main__":
    main()