chore: import upstream snapshot with attribution

eval/README.md

@@ -0,0 +1,5 @@
+# Gortex Eval Framework
+
+SWE-bench evaluation harness for measuring Gortex code intelligence impact on AI agent performance.
+
+See the [spec](.kiro/specs/eval-framework/) for full design details.

eval/__init__.py

@@ -0,0 +1 @@
+"""Gortex Eval Framework — benchmarking harness for measuring Gortex MCP tool impact."""

eval/agents/__init__.py

@@ -0,0 +1,5 @@
+"""Eval agent wrappers for different evaluation modes."""
+
+from agents.gortex_agent import GortexAgent, GortexMetrics, GortexMode
+
+__all__ = ["GortexAgent", "GortexMetrics", "GortexMode"]

eval/agents/gortex_agent.py

@@ -0,0 +1,347 @@
+"""Gortex-Enhanced Agent for SWE-bench Evaluation.
+
+Extends mini-swe-agent's DefaultAgent with Gortex code intelligence:
+
+1. **baseline** — bash only (grep, find, cat, sed). Control group.
+2. **native** — bash + Gortex tool bridge scripts via eval-server.
+3. **native_augment** — native + automatic grep output augmentation
+   with ``[Gortex]`` graph annotations (recommended).
+
+The agent is designed to work standalone even when mini-swe-agent is
+not installed — a lightweight base class is used as a fallback.
+
+Heavy lifting lives elsewhere:
+- Prompt selection: ``eval.prompts`` (system + instance templates per mode)
+- Augmentation pipeline: ``eval.augmentation`` (task 12)
+- Metrics persistence: ``eval.results`` (task 15)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger("gortex_agent")
+
+# ---------------------------------------------------------------------------
+# Try to import mini-swe-agent; fall back to a lightweight stub.
+# ---------------------------------------------------------------------------
+try:
+    from minisweagent.agents.default import DefaultAgent as _DefaultAgent
+except ImportError:  # pragma: no cover
+    class _DefaultAgent:  # type: ignore[no-redef]
+        """Minimal stand-in when mini-swe-agent is not installed."""
+
+        def __init__(self, **kwargs: Any) -> None:
+            self._kwargs = kwargs
+            self._step_count = 0
+
+        def run(self, task: str) -> dict:
+            raise NotImplementedError(
+                "mini-swe-agent is not installed — "
+                "install it or override run() in a subclass"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Gortex evaluation modes
+# ---------------------------------------------------------------------------
+
+class GortexMode(str, Enum):
+    """Evaluation modes for Gortex integration."""
+
+    BASELINE = "baseline"
+    NATIVE = "native"
+    NATIVE_AUGMENT = "native_augment"
+
+
+# ---------------------------------------------------------------------------
+# Tool bridge binaries → metric keys
+# ---------------------------------------------------------------------------
+
+# Maps a short metric key to the bash binary name installed in the container.
+TOOL_BINARIES: Dict[str, str] = {
+    "search_symbols": "gortex-search",
+    "smart_context": "gortex-context",
+    "explain_change_impact": "gortex-impact",
+    "graph_stats": "gortex-overview",
+    "find_usages": "gortex-usages",
+    "augment": "gortex-augment",
+}
+
+TOOL_METRIC_KEYS: List[str] = list(TOOL_BINARIES.keys())
+
+
+# ---------------------------------------------------------------------------
+# Metrics
+# ---------------------------------------------------------------------------
+
+@dataclass
+class GortexMetrics:
+    """Tracks Gortex-specific metrics during an evaluation run."""
+
+    tool_calls: Dict[str, int] = field(default_factory=lambda: {k: 0 for k in TOOL_METRIC_KEYS})
+    augmentation_calls: int = 0
+    augmentation_hits: int = 0
+    augmentation_errors: int = 0
+    augmentation_time_seconds: float = 0.0
+
+    @property
+    def total_tool_calls(self) -> int:
+        return sum(self.tool_calls.values())
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "tool_calls": dict(self.tool_calls),
+            "total_tool_calls": self.total_tool_calls,
+            "augmentation_calls": self.augmentation_calls,
+            "augmentation_hits": self.augmentation_hits,
+            "augmentation_errors": self.augmentation_errors,
+            "augmentation_time_seconds": round(self.augmentation_time_seconds, 2),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Pattern extraction (shared with augmentation pipeline)
+# ---------------------------------------------------------------------------
+
+_GREP_PATTERNS = [
+    # Quoted pattern: grep -rn "pattern" .
+    re.compile(r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*["\']([^"\']+)["\']'),
+    # Unquoted pattern: grep -rn pattern .
+    re.compile(r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*(\S+)'),
+]
+
+
+def extract_search_pattern(command: str) -> Optional[str]:
+    """Extract the search pattern from a grep/rg/ag command string.
+
+    Returns ``None`` when no usable pattern can be identified.
+    """
+    for pat in _GREP_PATTERNS:
+        match = pat.search(command)
+        if match:
+            result = match.group(1)
+            # Skip file paths and flags that were mis-captured
+            if result.startswith(("/", ".", "-")):
+                continue
+            return result
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Agent
+# ---------------------------------------------------------------------------
+
+class GortexAgent(_DefaultAgent):
+    """LLM agent with optional Gortex code-intelligence augmentation.
+
+    In **baseline** mode the agent behaves like a plain ``DefaultAgent``
+    with standard bash tools only.
+
+    In **native** mode the Gortex tool bridge scripts are available as
+    additional bash commands (``gortex-search``, ``gortex-context``, …).
+
+    In **native_augment** mode the agent additionally intercepts grep/rg
+    output and enriches it with ``[Gortex]`` graph annotations.
+
+    Parameters
+    ----------
+    config:
+        Merged run configuration dict (model + mode YAML).  The agent
+        reads ``config["agent"]`` for its own settings.
+    """
+
+    def __init__(self, config: Dict[str, Any], model: Any = None, env: Any = None, **kwargs: Any) -> None:
+        if model is not None and env is not None:
+            super().__init__(model=model, env=env, **kwargs)
+        else:
+            # Standalone mode (no mini-swe-agent)
+            self._kwargs = kwargs
+            self._step_count = 0
+
+        agent_cfg = config.get("agent", {})
+
+        # --- mode -----------------------------------------------------------
+        raw_mode = agent_cfg.get("gortex_mode", "baseline")
+        if isinstance(raw_mode, GortexMode):
+            self.mode = raw_mode
+        else:
+            self.mode = GortexMode(raw_mode)
+
+        # --- limits ----------------------------------------------------------
+        self.cost_limit: float = float(agent_cfg.get("cost_limit", 3.0))
+        self.step_limit: int = int(agent_cfg.get("step_limit", 30))
+
+        # --- augmentation settings -------------------------------------------
+        self.augment_timeout: float = float(agent_cfg.get("augment_timeout", 5.0))
+        self.augment_min_pattern_length: int = int(
+            agent_cfg.get("augment_min_pattern_length", 3)
+        )
+        self.track_gortex_usage: bool = bool(
+            agent_cfg.get("track_gortex_usage", True)
+        )
+
+        # --- prompt templates ------------------------------------------------
+        self._system_template = None
+        self._instance_template = None
+        self._load_prompt_templates()
+
+        # --- metrics ---------------------------------------------------------
+        self.metrics = GortexMetrics()
+
+        # --- internal bookkeeping --------------------------------------------
+        self._step_count = 0
+        self._total_cost: float = 0.0
+
+        logger.info(
+            "GortexAgent initialised: mode=%s, step_limit=%d, cost_limit=%.2f",
+            self.mode.value,
+            self.step_limit,
+            self.cost_limit,
+        )
+
+    # ------------------------------------------------------------------
+    # Prompt loading
+    # ------------------------------------------------------------------
+
+    def _load_prompt_templates(self) -> None:
+        """Load mode-specific Jinja2 prompt templates via ``eval.prompts``."""
+        try:
+            from prompts import load_templates
+
+            self._system_template, self._instance_template = load_templates(
+                self.mode.value
+            )
+            logger.debug("Loaded prompt templates for mode=%s", self.mode.value)
+        except Exception as exc:
+            logger.warning("Could not load prompt templates: %s", exc)
+
+    def render_system_prompt(self) -> str:
+        """Render the system prompt for the current mode."""
+        if self._system_template is None:
+            return ""
+        return self._system_template.render()
+
+    def render_instance_prompt(self, task: str) -> str:
+        """Render the instance prompt with the given *task* description."""
+        if self._instance_template is None:
+            return task
+        try:
+            from prompts import render_instance_prompt
+
+            return render_instance_prompt(self._instance_template, task)
+        except Exception:
+            return task
+
+    # ------------------------------------------------------------------
+    # Execution helpers
+    # ------------------------------------------------------------------
+
+    def should_continue(self) -> bool:
+        """Return ``False`` when a cost or step limit has been reached."""
+        if self._step_count >= self.step_limit:
+            logger.info("Step limit reached (%d)", self.step_limit)
+            return False
+        if self._total_cost >= self.cost_limit:
+            logger.info("Cost limit reached ($%.2f)", self.cost_limit)
+            return False
+        return True
+
+    def record_step(self, cost: float = 0.0) -> None:
+        """Record one agent step and its associated API cost."""
+        self._step_count += 1
+        self._total_cost += cost
+
+    # ------------------------------------------------------------------
+    # Tool-usage tracking
+    # ------------------------------------------------------------------
+
+    def track_tool_usage(self, command: str) -> None:
+        """Inspect *command* and increment the matching tool-call counter."""
+        if not self.track_gortex_usage:
+            return
+        for key, binary in TOOL_BINARIES.items():
+            if binary in command:
+                self.metrics.tool_calls[key] = self.metrics.tool_calls.get(key, 0) + 1
+                break
+
+    # ------------------------------------------------------------------
+    # Grep augmentation (native_augment mode)
+    # ------------------------------------------------------------------
+
+    def maybe_augment(
+        self,
+        command: str,
+        output: str,
+        *,
+        execute_fn: Any = None,
+    ) -> str:
+        """Conditionally augment grep/rg output with Gortex annotations.
+
+        In ``native_augment`` mode, if *command* is a grep/rg invocation
+        with a pattern of sufficient length, the augmentation endpoint is
+        called and ``[Gortex]`` annotations are appended.
+
+        Parameters
+        ----------
+        command:
+            The bash command that was executed.
+        output:
+            The raw stdout captured from the command.
+        execute_fn:
+            A callable ``(cmd: str, timeout: float) -> str`` that runs a
+            command inside the container and returns its stdout.  When
+            ``None``, augmentation is skipped.
+
+        Returns
+        -------
+        str
+            The (possibly enriched) output.
+        """
+        if self.mode != GortexMode.NATIVE_AUGMENT:
+            return output
+
+        pattern = extract_search_pattern(command)
+        if not pattern or len(pattern) < self.augment_min_pattern_length:
+            return output
+
+        if execute_fn is None:
+            return output
+
+        start = time.time()
+        try:
+            augment_result = execute_fn(
+                f'gortex-augment "{pattern}" 2>&1 || true',
+                self.augment_timeout,
+            )
+            elapsed = time.time() - start
+            self.metrics.augmentation_calls += 1
+            self.metrics.augmentation_time_seconds += elapsed
+
+            augment_text = (augment_result or "").strip()
+            if augment_text and "[Gortex]" in augment_text:
+                self.metrics.augmentation_hits += 1
+                return f"{output}\n\n{augment_text}"
+        except Exception as exc:
+            logger.debug("Augmentation failed for pattern '%s': %s", pattern, exc)
+            self.metrics.augmentation_errors += 1
+
+        return output
+
+    # ------------------------------------------------------------------
+    # Serialization
+    # ------------------------------------------------------------------
+
+    def get_metrics(self) -> Dict[str, Any]:
+        """Return a dict of Gortex-specific metrics for result storage."""
+        return {
+            "mode": self.mode.value,
+            "step_count": self._step_count,
+            "total_cost": round(self._total_cost, 4),
+            "gortex_metrics": self.metrics.to_dict(),
+        }

eval/analysis/__init__.py

@@ -0,0 +1 @@
+"""Post-run analysis and result comparison tools."""

eval/analysis/analyze_results.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""Results analyzer for the Gortex eval framework.
+
+Reads evaluation results and generates comparative analysis:
+- Summary table of patch rate, cost, tokens, duration per (model, mode)
+- Side-by-side mode comparison for a specific model
+- Gortex tool usage frequency and latency breakdown
+
+Usage:
+    python -m eval.analysis.analyze_results summary results/
+    python -m eval.analysis.analyze_results compare-modes results/ -m claude-sonnet
+    python -m eval.analysis.analyze_results tool-usage results/
+    python -m eval.analysis.analyze_results summary results/ --format csv
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import io
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from tabulate import tabulate
+
+from results import RunSummary
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+
+def _load_summaries(results_dir: Path) -> list[RunSummary]:
+    """Load summaries from *results_dir*, recomputing from per-instance files when available."""
+    summaries: list[RunSummary] = []
+    if not results_dir.is_dir():
+        return summaries
+    for run_dir in sorted(results_dir.iterdir()):
+        if not run_dir.is_dir():
+            continue
+
+        # Collect per-instance result files
+        instance_results = []
+        for inst_dir in sorted(run_dir.iterdir()):
+            if not inst_dir.is_dir():
+                continue
+            for json_file in inst_dir.glob("*.json"):
+                if "_trajectory" in json_file.name:
+                    continue
+                try:
+                    instance_results.append(json.loads(json_file.read_text()))
+                except Exception:
+                    pass
+
+        if not instance_results:
+            # Fall back to summary.json
+            summary_path = run_dir / "summary.json"
+            if summary_path.exists():
+                try:
+                    data = json.loads(summary_path.read_text())
+                    summaries.append(RunSummary.from_dict(data))
+                except Exception:
+                    pass
+            continue
+
+        # Recompute summary from per-instance data
+        total = len(instance_results)
+        patches = sum(1 for r in instance_results if r.get("submission"))
+        total_cost = sum(r.get("cost", 0) for r in instance_results)
+        total_tokens = sum(r.get("tokens_input", 0) + r.get("tokens_output", 0) for r in instance_results)
+        total_duration = sum(r.get("duration_seconds", 0) for r in instance_results)
+        completed = sum(1 for r in instance_results if r.get("exit_status") not in (None, "error", "setup_failure"))
+
+        model = instance_results[0].get("model", "")
+        mode = instance_results[0].get("mode", "")
+
+        summaries.append(RunSummary(
+            run_id=run_dir.name,
+            model=model,
+            mode=mode,
+            total_instances=total,
+            completed=completed,
+            patch_rate=patches / total if total else 0,
+            total_cost=total_cost,
+            mean_cost=total_cost / total if total else 0,
+            total_tokens=total_tokens,
+            mean_tokens=total_tokens / total if total else 0,
+            total_duration_seconds=total_duration,
+            mean_duration_seconds=total_duration / total if total else 0,
+        ))
+
+    return summaries
+
+
+def _load_instance_results(results_dir: Path) -> list[dict[str, Any]]:
+    """Load all per-instance JSON result files from *results_dir*."""
+    instances: list[dict[str, Any]] = []
+    if not results_dir.is_dir():
+        return instances
+    for run_dir in sorted(results_dir.iterdir()):
+        if not run_dir.is_dir():
+            continue
+        for inst_dir in sorted(run_dir.iterdir()):
+            if not inst_dir.is_dir():
+                continue
+            for json_file in inst_dir.glob("*.json"):
+                try:
+                    instances.append(json.loads(json_file.read_text()))
+                except Exception:
+                    pass
+    return instances
+
+
+# ---------------------------------------------------------------------------
+# Output helpers
+# ---------------------------------------------------------------------------
+
+
+
+def _output(headers: list[str], rows: list[list[Any]], fmt: str) -> None:
+    """Print *rows* with *headers* in the requested format."""
+    if fmt == "csv":
+        buf = io.StringIO()
+        writer = csv.writer(buf)
+        writer.writerow(headers)
+        writer.writerows(rows)
+        sys.stdout.write(buf.getvalue())
+    else:
+        print(tabulate(rows, headers=headers, tablefmt="grid"))
+
+
+# ---------------------------------------------------------------------------
+# Commands
+# ---------------------------------------------------------------------------
+
+
+def summary(results_dir: str, fmt: str = "table", swebench_eval: bool = False) -> None:
+    """Table of patch rate, mean cost, mean tokens, mean duration per (model, mode)."""
+    summaries = _load_summaries(Path(results_dir))
+    if not summaries:
+        print(f"No results found in {results_dir}")
+        return
+
+    if swebench_eval:
+        print(
+            "NOTE: --swebench-eval is a placeholder. "
+            "To run the official SWE-bench test harness, install the swebench "
+            "package and invoke:\n"
+            "  python -m swebench.harness.run_evaluation "
+            "--predictions_path <results>/<run_id>/preds.json "
+            "--dataset_name princeton-nlp/SWE-Bench_Lite"
+        )
+        print()
+
+    headers = ["Model", "Mode", "Instances", "Patch Rate", "Mean Cost", "Mean Tokens", "Mean Duration (s)"]
+    rows: list[list[Any]] = []
+    for s in summaries:
+        rows.append([
+            s.model,
+            s.mode,
+            s.total_instances,
+            f"{s.patch_rate:.1%}",
+            f"${s.mean_cost:.4f}",
+            f"{s.mean_tokens:.0f}",
+            f"{s.mean_duration_seconds:.1f}",
+        ])
+
+    _output(headers, rows, fmt)
+
+
+def compare_modes(results_dir: str, model: str, fmt: str = "table") -> None:
+    """Side-by-side baseline vs native vs native_augment with deltas for *model*."""
+    summaries = _load_summaries(Path(results_dir))
+    model_runs = {s.mode: s for s in summaries if s.model == model}
+
+    if not model_runs:
+        print(f"No results found for model: {model}")
+        return
+
+    mode_order = [m for m in ("baseline", "native", "native_augment") if m in model_runs]
+    mode_order += sorted(set(model_runs) - set(mode_order))
+
+    metrics = ["patch_rate", "mean_cost", "mean_tokens", "mean_duration_seconds"]
+    metric_labels = ["Patch Rate", "Mean Cost ($)", "Mean Tokens", "Mean Duration (s)"]
+
+    headers = ["Metric"] + mode_order
+    # Add delta columns if baseline exists
+    baseline = model_runs.get("baseline")
+    if baseline:
+        for m in mode_order:
+            if m != "baseline":
+                headers.append(f"Δ {m} vs baseline")
+
+    rows: list[list[Any]] = []
+    for label, attr in zip(metric_labels, metrics):
+        row: list[Any] = [label]
+        values: dict[str, float] = {}
+        for mode in mode_order:
+            s = model_runs[mode]
+            v = getattr(s, attr, 0.0)
+            values[mode] = v
+            if attr == "patch_rate":
+                row.append(f"{v:.1%}")
+            elif attr == "mean_cost":
+                row.append(f"${v:.4f}")
+            else:
+                row.append(f"{v:.1f}")
+
+        if baseline:
+            bv = values.get("baseline", 0.0)
+            for mode in mode_order:
+                if mode == "baseline":
+                    continue
+                mv = values[mode]
+                if bv != 0:
+                    delta_pct = ((mv - bv) / abs(bv)) * 100
+                    row.append(f"{delta_pct:+.1f}%")
+                else:
+                    row.append("N/A")
+
+        rows.append(row)
+
+    print(f"\nMode comparison for model: {model}\n")
+    _output(headers, rows, fmt)
+
+
+def tool_usage(results_dir: str, fmt: str = "table") -> None:
+    """Gortex tool call frequency and latency breakdown per tool name."""
+    instances = _load_instance_results(Path(results_dir))
+    if not instances:
+        print(f"No instance results found in {results_dir}")
+        return
+
+    # Aggregate tool calls across all instances
+    tool_counts: dict[str, int] = {}
+    tool_latencies: dict[str, list[float]] = {}
+
+    for inst in instances:
+        gm = inst.get("gortex_metrics", {})
+        calls = gm.get("tool_calls", {})
+        for tool_name, count in calls.items():
+            tool_counts[tool_name] = tool_counts.get(tool_name, 0) + count
+
+        # If per-tool latencies are available
+        latencies = gm.get("tool_latencies", {})
+        for tool_name, lat in latencies.items():
+            tool_latencies.setdefault(tool_name, []).append(lat)
+
+    if not tool_counts:
+        print("No Gortex tool usage data found.")
+        return
+
+    headers = ["Tool", "Total Calls", "Mean Latency (s)"]
+    rows: list[list[Any]] = []
+    for tool_name in sorted(tool_counts):
+        count = tool_counts[tool_name]
+        lats = tool_latencies.get(tool_name, [])
+        mean_lat = f"{sum(lats) / len(lats):.3f}" if lats else "N/A"
+        rows.append([tool_name, count, mean_lat])
+
+    _output(headers, rows, fmt)
+
+
+# ---------------------------------------------------------------------------
+# CLI (argparse)
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="analyze_results",
+        description="Post-run analysis for Gortex eval results.",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["csv", "table"],
+        default="table",
+        help="Output format (default: table)",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # summary
+    sp_summary = subparsers.add_parser("summary", help="Summary table per (model, mode)")
+    sp_summary.add_argument("results_dir", help="Path to results directory")
+    sp_summary.add_argument(
+        "--swebench-eval",
+        action="store_true",
+        default=False,
+        help="Run official SWE-bench test harness on collected patches",
+    )
+
+    # compare-modes
+    sp_compare = subparsers.add_parser("compare-modes", help="Side-by-side mode comparison")
+    sp_compare.add_argument("results_dir", help="Path to results directory")
+    sp_compare.add_argument("-m", "--model", required=True, help="Model to compare across modes")
+
+    # tool-usage
+    sp_tools = subparsers.add_parser("tool-usage", help="Gortex tool call frequency and latency")
+    sp_tools.add_argument("results_dir", help="Path to results directory")
+
+    args = parser.parse_args()
+
+    if args.command == "summary":
+        summary(args.results_dir, fmt=args.format, swebench_eval=args.swebench_eval)
+    elif args.command == "compare-modes":
+        compare_modes(args.results_dir, model=args.model, fmt=args.format)
+    elif args.command == "tool-usage":
+        tool_usage(args.results_dir, fmt=args.format)
+
+
+if __name__ == "__main__":
+    main()

eval/augmentation.py

@@ -0,0 +1,139 @@
+"""Augmentation pipeline for grep/rg output enrichment.
+
+In ``native_augment`` mode, grep/rg output is post-processed with Gortex
+graph annotations (callers, callees, execution flows) via the eval-server's
+``/augment`` endpoint.
+
+Feature: eval-framework
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional
+
+from agents.gortex_agent import extract_search_pattern
+
+logger = logging.getLogger("gortex_augmentation")
+
+# Defaults matching native_augment.yaml
+_DEFAULT_TIMEOUT: float = 5.0
+_DEFAULT_MIN_PATTERN_LENGTH: int = 3
+
+
+def augment_grep_output(
+    raw_output: str,
+    command: str,
+    eval_server_url: str,
+    config: Optional[Dict[str, Any]] = None,
+) -> str:
+    """Augment grep/rg output with Gortex graph annotations.
+
+    Parameters
+    ----------
+    raw_output:
+        The raw stdout captured from the grep/rg command.
+    command:
+        The bash command string that was executed.
+    eval_server_url:
+        Base URL of the eval-server (e.g. ``http://127.0.0.1:4747``).
+    config:
+        Optional config dict. Reads ``augment_timeout`` and
+        ``augment_min_pattern_length`` from it.
+
+    Returns
+    -------
+    str
+        The (possibly enriched) output. Returns *raw_output* unmodified
+        when augmentation is skipped, times out, or returns nothing useful.
+    """
+    cfg = config or {}
+    timeout = float(cfg.get("augment_timeout", _DEFAULT_TIMEOUT))
+    min_len = int(cfg.get("augment_min_pattern_length", _DEFAULT_MIN_PATTERN_LENGTH))
+
+    # Extract search pattern from the command string.
+    pattern = extract_search_pattern(command)
+    if pattern is None or len(pattern) < min_len:
+        return raw_output
+
+    # POST to /augment endpoint.
+    url = f"{eval_server_url.rstrip('/')}/augment"
+    payload = json.dumps({"pattern": pattern}).encode("utf-8")
+
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            body = resp.read().decode("utf-8")
+    except (urllib.error.URLError, OSError, TimeoutError):
+        # Timeout or connection error — return original output.
+        return raw_output
+
+    # Parse response and format annotations.
+    try:
+        data = json.loads(body)
+    except (json.JSONDecodeError, ValueError):
+        return raw_output
+
+    annotations = _format_annotations(data)
+    if not annotations:
+        return raw_output
+
+    return f"{raw_output}\n{annotations}"
+
+
+def _format_annotations(data: Dict[str, Any]) -> str:
+    """Format augmentation response data as ``[Gortex]`` annotation lines.
+
+    The eval-server ``/augment`` endpoint returns a dict with optional keys:
+    ``callers``, ``callees``, ``flows`` — each a list of
+    ``{"name": ..., "location": ...}`` dicts.
+
+    Returns an empty string when there is nothing useful to annotate.
+    """
+    lines: list[str] = []
+
+    callers = data.get("callers")
+    if callers and isinstance(callers, list):
+        caller_strs = [_format_ref(c) for c in callers if _format_ref(c)]
+        if caller_strs:
+            lines.append(f"  [Gortex] callers: {', '.join(caller_strs)}")
+
+    callees = data.get("callees")
+    if callees and isinstance(callees, list):
+        callee_strs = [_format_ref(c) for c in callees if _format_ref(c)]
+        if callee_strs:
+            lines.append(f"  [Gortex] callees: {', '.join(callee_strs)}")
+
+    flows = data.get("flows")
+    if flows and isinstance(flows, list):
+        flow_strs = [_format_ref(f) for f in flows if _format_ref(f)]
+        if flow_strs:
+            lines.append(f"  [Gortex] flows: {', '.join(flow_strs)}")
+
+    return "\n".join(lines)
+
+
+def _format_ref(ref: Any) -> str:
+    """Format a single caller/callee/flow reference.
+
+    Accepts either a dict with ``name`` and optional ``location`` keys,
+    or a plain string.
+    """
+    if isinstance(ref, str):
+        return ref
+    if isinstance(ref, dict):
+        name = ref.get("name", "")
+        location = ref.get("location", "")
+        if name and location:
+            return f"{name} ({location})"
+        return name or location or ""
+    return ""

eval/bridge/__init__.py

@@ -0,0 +1 @@
+"""Tool bridge utilities for Gortex MCP tool exposure."""

eval/bridge/gortex-augment

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# gortex-augment — grep output augmentation helper via eval-server
+# Usage: gortex-augment <search_pattern> [raw_output]
+# Internal helper for native_augment mode — enriches grep results with graph context.
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+AUGMENT_TIMEOUT="${GORTEX_AUGMENT_TIMEOUT:-5}"
+
+pattern="${1:-}"
+raw_output="${2:-}"
+
+if [ -z "$pattern" ]; then
+    echo "Usage: gortex-augment <search_pattern> [raw_output]"
+    echo "Enrich grep/rg output with graph annotations (callers, callees, flows)."
+    echo "Typically called automatically in native_augment mode."
+    echo ""
+    echo "Examples:"
+    echo '  gortex-augment "ValidateToken"'
+    echo '  grep -rn "ValidateToken" src/ | gortex-augment "ValidateToken" "$(cat)"'
+    exit 1
+fi
+
+# Skip augmentation for very short patterns
+if [ "${#pattern}" -lt "${GORTEX_AUGMENT_MIN_PATTERN:-3}" ]; then
+    [ -n "$raw_output" ] && echo "$raw_output"
+    exit 0
+fi
+
+# Escape pattern for JSON
+escaped_pattern=$(printf '%s' "$pattern" | sed 's/\\/\\\\/g; s/"/\\"/g')
+payload="{\"pattern\":\"$escaped_pattern\""
+if [ -n "$raw_output" ]; then
+    escaped_output=$(printf '%s' "$raw_output" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | head -c 8192)
+    payload="$payload,\"raw_output\":\"$escaped_output\""
+fi
+payload="$payload}"
+
+# Try eval-server with timeout (augmentation must be fast)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/augment" \
+    -H "Content-Type: application/json" \
+    --max-time "$AUGMENT_TIMEOUT" \
+    -d "$payload" 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    if command -v jq &>/dev/null; then
+        augmented=$(echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif .augmented_output then
+                .augmented_output
+            elif type == "object" and .error then
+                empty
+            else
+                tostring
+            end
+        ' 2>/dev/null)
+
+        if [ -n "$augmented" ]; then
+            echo "$augmented"
+        elif [ -n "$raw_output" ]; then
+            echo "$raw_output"
+        fi
+    else
+        # Without jq, try to use the response directly
+        if echo "$response" | grep -q '"error"'; then
+            [ -n "$raw_output" ] && echo "$raw_output"
+        else
+            echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+        fi
+    fi
+else
+    # Timeout or server unavailable — return original output unmodified
+    [ -n "$raw_output" ] && echo "$raw_output"
+fi

eval/bridge/gortex-context

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+# gortex-context — smart context via eval-server
+# Usage: gortex-context <task_description> [entry_point] [max_symbols]
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+
+task="${1:-}"
+entry_point="${2:-}"
+max_symbols="${3:-5}"
+
+if [ -z "$task" ]; then
+    echo "Usage: gortex-context <task_description> [entry_point] [max_symbols]"
+    echo "Get compact context for a task: relevant files, symbols, relationships."
+    echo ""
+    echo "Examples:"
+    echo '  gortex-context "fix authentication timeout bug"'
+    echo '  gortex-context "add rate limiting" "api/handler.go::HandleRequest"'
+    echo '  gortex-context "refactor database layer" "" 10'
+    exit 1
+fi
+
+# Build JSON payload
+payload="{\"task\":\"$task\",\"max_symbols\":$max_symbols"
+[ -n "$entry_point" ] && payload="$payload,\"entry_point\":\"$entry_point\""
+payload="$payload}"
+
+# Try eval-server (fast path)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/tool/smart_context" \
+    -H "Content-Type: application/json" \
+    -d "$payload" 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    if command -v jq &>/dev/null; then
+        echo "=== Smart Context: $task ==="
+        echo ""
+        # Extract the text content from the MCP response, then parse the inner
+        # JSON and produce a compact summary — dropping source code to save tokens.
+        echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif type == "object" and .error then
+                "Error: \(.error)"
+            else
+                tostring
+            end
+        ' 2>/dev/null | jq -r '
+            # Parse the smart_context JSON blob and produce compact output.
+            if type != "object" then . else
+
+            # Files to edit
+            (if .files_to_edit then
+                "Files to edit:\n" + (.files_to_edit | map("  " + .) | join("\n"))
+            else empty end),
+
+            # Relevant symbols (compact: signature only, no source)
+            (if .relevant_symbols then
+                "\nRelevant symbols:\n" + (
+                    .relevant_symbols | map(
+                        "  " + .kind + " " + .name + " " + .file_path + ":" + (.start_line // 0 | tostring)
+                        + (if .signature then " — " + .signature else "" end)
+                    ) | join("\n")
+                )
+            else empty end),
+
+            # Callers
+            (if .callers then
+                "\nCallers:\n" + (.callers | map("  " + .) | join("\n"))
+            else empty end),
+
+            # Callees
+            (if .callees then
+                "\nCallees:\n" + (.callees | map("  " + .) | join("\n"))
+            else empty end),
+
+            # Related test files
+            (if .related_test_files and (.related_test_files | length) > 0 then
+                "\nTest files:\n" + (.related_test_files | map("  " + .) | join("\n"))
+            else empty end),
+
+            # Cross-repo deps (compact)
+            (if .cross_repo_dependencies then
+                "\nCross-repo deps:\n" + (
+                    .cross_repo_dependencies | map(
+                        "  " + .edge_kind + " " + .name + " (" + .repo_prefix + ")"
+                    ) | join("\n")
+                )
+            else empty end),
+
+            # Keywords used
+            (if .keywords then
+                "\nKeywords: " + (.keywords | join(", "))
+            else empty end)
+
+            end
+        ' 2>/dev/null || {
+            # If inner jq parse fails (non-JSON text), just print the raw text
+            echo "$response" | jq -r '
+                if .content then
+                    .content[] | select(.type == "text") | .text
+                else tostring end
+            ' 2>/dev/null || echo "$response"
+        }
+    else
+        echo "=== Smart Context: $task ==="
+        echo ""
+        echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+    fi
+else
+    # Fallback: gortex CLI
+    echo "Error: Gortex eval-server not running on port $GORTEX_EVAL_PORT. Start it with: gortex eval-server --index /testbed" >&2
+    if false; then
+        gortex smart-context --task "$task" ${entry_point:+--entry-point "$entry_point"} --max-symbols "$max_symbols" 2>&1
+    else
+        echo "Error: Gortex tools temporarily unavailable (no eval-server, no CLI)." >&2
+        exit 1
+    fi
+fi
+
+echo ""
+echo "--- Next steps ---"
+echo "  gortex-search \"<pattern>\"     — search for more symbols related to the task"
+echo "  gortex-impact \"<symbol_id>\"   — check blast radius before making changes"
+echo "  gortex-usages \"<symbol_id>\"   — find all callers of a symbol you plan to modify"

eval/bridge/gortex-impact

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# gortex-impact — blast radius analysis via eval-server
+# Usage: gortex-impact <symbol_ids>
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+
+symbol_ids="${1:-}"
+
+if [ -z "$symbol_ids" ]; then
+    echo "Usage: gortex-impact <symbol_ids>"
+    echo "Analyze the blast radius of changing one or more symbols."
+    echo "Pass comma-separated symbol IDs for multiple symbols."
+    echo ""
+    echo "Examples:"
+    echo '  gortex-impact "internal/auth/service.go::ValidateToken"'
+    echo '  gortex-impact "api/handler.go::HandleRequest,internal/auth/service.go::ValidateToken"'
+    exit 1
+fi
+
+payload=$(printf '{"symbol_ids":"%s"}' "$symbol_ids")
+
+# Try eval-server (fast path)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/tool/explain_change_impact" \
+    -H "Content-Type: application/json" \
+    -d "$payload" 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    if command -v jq &>/dev/null; then
+        echo "=== Change Impact: $symbol_ids ==="
+        echo ""
+        echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif type == "object" and .error then
+                "Error: \(.error)"
+            else
+                tostring
+            end
+        ' 2>/dev/null || echo "$response"
+    else
+        echo "=== Change Impact: $symbol_ids ==="
+        echo ""
+        echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+    fi
+else
+    # Fallback: gortex CLI
+    echo "Error: Gortex eval-server not running on port $GORTEX_EVAL_PORT. Start it with: gortex eval-server --index /testbed" >&2
+    if false; then
+        gortex impact --symbols "$symbol_ids" 2>&1
+    else
+        echo "Error: Gortex tools temporarily unavailable (no eval-server, no CLI)." >&2
+        exit 1
+    fi
+fi
+
+echo ""
+echo "--- Next steps ---"
+echo "  gortex-usages \"<symbol_id>\"     — find all references to update after your change"
+echo "  gortex-context \"<fix task>\"      — get full context and edit plan for the fix"
+echo "  gortex-overview                   — check overall graph stats and health"

eval/bridge/gortex-overview

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# gortex-overview — graph statistics and health via eval-server
+# Usage: gortex-overview
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+
+# Try eval-server (fast path)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/tool/graph_stats" \
+    -H "Content-Type: application/json" \
+    -d '{}' 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    if command -v jq &>/dev/null; then
+        echo "=== Codebase Overview ==="
+        echo ""
+        echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif type == "object" and .error then
+                "Error: \(.error)"
+            else
+                tostring
+            end
+        ' 2>/dev/null || echo "$response"
+    else
+        echo "=== Codebase Overview ==="
+        echo ""
+        echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+    fi
+else
+    # Fallback: gortex CLI
+    echo "Error: Gortex eval-server not running on port $GORTEX_EVAL_PORT. Start it with: gortex eval-server --index /testbed" >&2
+    if false; then
+        gortex stats 2>&1
+    else
+        echo "Error: Gortex tools temporarily unavailable (no eval-server, no CLI)." >&2
+        exit 1
+    fi
+fi
+
+echo ""
+echo "--- Next steps ---"
+echo "  gortex-search \"<query>\"        — search for symbols related to your task"
+echo "  gortex-context \"<task>\"         — get full context for a specific task"
+echo "  gortex-impact \"<symbol_id>\"     — analyze blast radius of a planned change"

eval/bridge/gortex-search

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# gortex-search — BM25 symbol search via eval-server
+# Usage: gortex-search <query> [limit]
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+
+query="${1:-}"
+limit="${2:-20}"
+
+if [ -z "$query" ]; then
+    echo "Usage: gortex-search <query> [limit]"
+    echo "Search the codebase for symbols matching a query (BM25 + camelCase-aware)."
+    echo ""
+    echo "Examples:"
+    echo '  gortex-search "validate token"'
+    echo '  gortex-search "HandleRequest" 10'
+    exit 1
+fi
+
+payload=$(printf '{"query":"%s","limit":%s,"compact":true}' "$query" "$limit")
+
+# Try eval-server (fast path — graph stays warm in memory)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/tool/search_symbols" \
+    -H "Content-Type: application/json" \
+    -d "$payload" 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    # Format JSON response as plain text
+    if command -v jq &>/dev/null; then
+        echo "=== Symbol Search: $query ==="
+        echo ""
+        echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif type == "array" then
+                .[] | if .id then "\(.id)  \(.kind // "")  \(.file // "")" else tostring end
+            elif type == "object" and .error then
+                "Error: \(.error)"
+            else
+                tostring
+            end
+        ' 2>/dev/null || echo "$response"
+    else
+        echo "=== Symbol Search: $query ==="
+        echo ""
+        echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+    fi
+else
+    # Fallback: gortex CLI
+    echo "Error: Gortex eval-server not running on port $GORTEX_EVAL_PORT. Start it with: gortex eval-server --index /testbed" >&2
+    if false; then
+        gortex search "$query" --limit "$limit" --compact 2>&1
+    else
+        echo "Error: Gortex tools temporarily unavailable (no eval-server, no CLI)." >&2
+        exit 1
+    fi
+fi
+
+echo ""
+echo "--- Next steps ---"
+echo "  gortex-context \"<task description>\"   — get full context for a task (callers, callees, edit plan)"
+echo "  gortex-usages \"<symbol_id>\"           — find all references to a symbol"
+echo "  gortex-impact \"<symbol_id>\"           — analyze blast radius before changing a symbol"

eval/bridge/gortex-usages

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# gortex-usages — find all references to a symbol via eval-server
+# Usage: gortex-usages <symbol_id> [limit]
+set -euo pipefail
+
+GORTEX_EVAL_PORT="${GORTEX_EVAL_PORT:-4747}"
+GORTEX_EVAL_URL="http://127.0.0.1:${GORTEX_EVAL_PORT}"
+
+symbol_id="${1:-}"
+limit="${2:-50}"
+
+if [ -z "$symbol_id" ]; then
+    echo "Usage: gortex-usages <symbol_id> [limit]"
+    echo "Find all references to a symbol across the codebase (zero false positives)."
+    echo ""
+    echo "Examples:"
+    echo '  gortex-usages "internal/auth/service.go::ValidateToken"'
+    echo '  gortex-usages "api/handler.go::HandleRequest" 20'
+    exit 1
+fi
+
+payload=$(printf '{"id":"%s","limit":%s,"compact":true}' "$symbol_id" "$limit")
+
+# Try eval-server (fast path)
+response=$(curl -sf -X POST "${GORTEX_EVAL_URL}/tool/find_usages" \
+    -H "Content-Type: application/json" \
+    -d "$payload" 2>/dev/null) && server_ok=true || server_ok=false
+
+if [ "$server_ok" = true ] && [ -n "$response" ]; then
+    if command -v jq &>/dev/null; then
+        echo "=== Usages: $symbol_id ==="
+        echo ""
+        echo "$response" | jq -r '
+            if .content then
+                .content[] | select(.type == "text") | .text
+            elif type == "object" and .error then
+                "Error: \(.error)"
+            else
+                tostring
+            end
+        ' 2>/dev/null || echo "$response"
+    else
+        echo "=== Usages: $symbol_id ==="
+        echo ""
+        echo "$response" | sed 's/[{}\[\]"]//g; s/,/\n/g' | grep -v '^\s*$'
+    fi
+else
+    # Fallback: gortex CLI
+    echo "Error: Gortex eval-server not running on port $GORTEX_EVAL_PORT. Start it with: gortex eval-server --index /testbed" >&2
+    if false; then
+        gortex usages --id "$symbol_id" --limit "$limit" --compact 2>&1
+    else
+        echo "Error: Gortex tools temporarily unavailable (no eval-server, no CLI)." >&2
+        exit 1
+    fi
+fi
+
+echo ""
+echo "--- Next steps ---"
+echo "  gortex-impact \"$symbol_id\"     — check blast radius before changing this symbol"
+echo "  gortex-context \"fix <task>\"     — get full context and edit plan"
+echo "  gortex-search \"<related>\"       — search for related symbols"

eval/config.py

@@ -0,0 +1,100 @@
+"""Config loading, merging, and validation for the eval framework.
+
+Loads model and mode YAML configs from eval/configs/, merges them with
+mode overriding model on key conflicts, and validates required fields.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Resolve configs directory relative to this file (eval/configs/)
+_CONFIGS_DIR = Path(__file__).resolve().parent / "configs"
+
+
+def load_model_config(name: str) -> dict[str, Any]:
+    """Load a model config from configs/models/{name}.yaml."""
+    path = _CONFIGS_DIR / "models" / f"{name}.yaml"
+    if not path.is_file():
+        raise FileNotFoundError(f"Model config not found: {path}")
+    with open(path) as f:
+        return yaml.safe_load(f) or {}
+
+
+def load_mode_config(name: str) -> dict[str, Any]:
+    """Load a mode config from configs/modes/{name}.yaml."""
+    path = _CONFIGS_DIR / "modes" / f"{name}.yaml"
+    if not path.is_file():
+        raise FileNotFoundError(f"Mode config not found: {path}")
+    with open(path) as f:
+        return yaml.safe_load(f) or {}
+
+
+def merge_configs(model_config: dict[str, Any], mode_config: dict[str, Any]) -> dict[str, Any]:
+    """Deep-merge two config dicts. Mode values override model on key conflicts.
+
+    For nested dicts, merges recursively. For scalar values, mode wins.
+    """
+    return _deep_merge(model_config, mode_config)
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    """Recursively merge override into base. Override wins on conflicts."""
+    result = dict(base)
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+_REQUIRED_FIELDS = [
+    "model.model_name",
+    "agent.agent_class",
+    "environment.environment_class",
+]
+
+
+def validate_config(merged: dict[str, Any]) -> None:
+    """Validate that all required fields are present in the merged config.
+
+    Required fields use dot notation: 'model.model_name' means
+    merged["model"]["model_name"].
+
+    Raises ValueError listing all missing fields if any are absent.
+    """
+    missing = []
+    for field in _REQUIRED_FIELDS:
+        parts = field.split(".")
+        current = merged
+        found = True
+        for part in parts:
+            if not isinstance(current, dict) or part not in current:
+                found = False
+                break
+            current = current[part]
+        if not found:
+            missing.append(field)
+
+    if missing:
+        raise ValueError(f"Missing required config fields: {', '.join(missing)}")
+
+
+def list_configs() -> dict[str, list[str]]:
+    """Discover all YAML config files in configs/models/ and configs/modes/.
+
+    Returns a dict with 'models' and 'modes' keys, each containing a list
+    of config names (without the .yaml extension).
+    """
+    result: dict[str, list[str]] = {"models": [], "modes": []}
+    for category in ("models", "modes"):
+        directory = _CONFIGS_DIR / category
+        if directory.is_dir():
+            result[category] = sorted(
+                p.stem for p in directory.iterdir() if p.suffix in (".yaml", ".yml") and p.is_file()
+            )
+    return result

eval/configs/models/.gitkeep

@@ -0,0 +1 @@
+

eval/configs/models/claude-haiku.yaml

@@ -0,0 +1,9 @@
+# Claude Haiku 3.5 — fast, cheap, good baseline
+# Via Anthropic directly (set ANTHROPIC_API_KEY in .env)
+# For OpenRouter, change to: openrouter/anthropic/claude-3.5-haiku
+model:
+  model_name: 'anthropic/claude-3-5-haiku-20241022'
+  cost_tracking: 'ignore_errors'
+  model_kwargs:
+    max_tokens: 8192
+    temperature: 0

eval/configs/models/claude-sonnet.yaml

@@ -0,0 +1,9 @@
+# Claude Sonnet 4 — strong all-around model
+# Via Anthropic directly (set ANTHROPIC_API_KEY in .env)
+# For OpenRouter, change to: openrouter/anthropic/claude-sonnet-4
+model:
+  model_name: 'anthropic/claude-sonnet-4-20250514'
+  cost_tracking: 'ignore_errors'
+  model_kwargs:
+    max_tokens: 16384
+    temperature: 0

eval/configs/modes/.gitkeep

@@ -0,0 +1 @@
+

eval/configs/modes/baseline.yaml

@@ -0,0 +1,12 @@
+# Baseline mode — no Gortex, pure bash tools (control group)
+agent:
+  agent_class: 'eval.agents.gortex_agent.GortexAgent'
+  gortex_mode: 'baseline'
+  step_limit: 30
+  cost_limit: 3.0
+
+environment:
+  environment_class: 'eval.environments.gortex_docker.GortexDockerEnvironment'
+  enable_gortex: false
+  # Set to 'podman' if using podman instead of docker
+  container_executable: 'podman'

eval/configs/modes/native.yaml

@@ -0,0 +1,22 @@
+# Native mode — Gortex tools only, no grep augmentation
+#
+# Explicit tools: gortex-search, gortex-context, gortex-impact, gortex-overview, gortex-usages
+# Available as fast bash commands (~100ms via eval-server)
+#
+# Use this mode to isolate the value of explicit tools without grep augmentation.
+agent:
+  agent_class: 'eval.agents.gortex_agent.GortexAgent'
+  gortex_mode: 'native'
+  step_limit: 30
+  cost_limit: 3.0
+  track_gortex_usage: true
+
+environment:
+  environment_class: 'eval.environments.gortex_docker.GortexDockerEnvironment'
+  enable_gortex: true
+  gortex_timeout: 120
+  eval_server_port: 4747
+  # Set to 'podman' if using podman instead of docker
+  container_executable: 'podman'
+  # Path to linux/amd64 gortex binary for container injection
+  gortex_binary: '../gortex-linux'

eval/configs/modes/native_augment.yaml

@@ -0,0 +1,27 @@
+# Native + Augment mode — the primary evaluation mode
+#
+# Combines two capabilities:
+# 1. Explicit Gortex tools: gortex-search, gortex-context, gortex-impact, gortex-overview, gortex-usages
+#    Available as fast bash commands (~100ms via eval-server)
+# 2. Automatic grep enrichment: grep/rg results are transparently augmented with
+#    [Gortex] annotations showing callers, callees, and execution flows
+#
+# The agent decides when to use explicit tools vs rely on enriched grep results.
+agent:
+  agent_class: 'eval.agents.gortex_agent.GortexAgent'
+  gortex_mode: 'native_augment'
+  step_limit: 30
+  cost_limit: 3.0
+  augment_timeout: 5.0
+  augment_min_pattern_length: 3
+  track_gortex_usage: true
+
+environment:
+  environment_class: 'eval.environments.gortex_docker.GortexDockerEnvironment'
+  enable_gortex: true
+  gortex_timeout: 120
+  eval_server_port: 4747
+  # Set to 'podman' if using podman instead of docker
+  container_executable: 'podman'
+  # Path to linux/amd64 gortex binary for container injection
+  gortex_binary: './gortex-linux'

eval/environments/__init__.py

@@ -0,0 +1 @@
+"""Docker-based sandboxed execution environments."""

eval/environments/gortex_docker.py

@@ -0,0 +1,432 @@
+"""Gortex Docker Environment for SWE-bench Evaluation.
+
+Manages the full container lifecycle for a single eval instance:
+  1. Launch container with target repo at specified commit
+  2. Copy gortex binary and tool bridge scripts (native/native_augment modes)
+  3. Start eval-server inside container, health-check with configurable timeout
+  4. Extract patch (git diff) before teardown
+  5. Mount/copy cached indexes when available
+  6. Record setup failures gracefully — never raise, return failure result
+
+Architecture:
+  Agent bash cmd → /usr/local/bin/gortex-search → curl localhost:4747/tool/search_symbols
+    → eval-server → in-memory graph
+  Fallback: → gortex CLI (cold path)
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import tarfile
+import time
+from pathlib import Path
+from typing import Any
+
+import docker
+
+logger = logging.getLogger("gortex_docker")
+
+DEFAULT_EVAL_SERVER_PORT = 4747
+DEFAULT_GORTEX_TIMEOUT = 120
+DEFAULT_CACHE_DIR = Path.home() / ".gortex-eval-cache"
+HEALTH_CHECK_INTERVAL = 2.0
+CONTAINER_WORKDIR = "/testbed"
+GORTEX_BINARY_CONTAINER_PATH = "/usr/local/bin/gortex"
+BRIDGE_SCRIPTS_CONTAINER_DIR = "/usr/local/bin"
+
+# Bridge script names (files in eval/bridge/ without __init__.py and __pycache__)
+_BRIDGE_SCRIPTS = [
+    "gortex-search",
+    "gortex-context",
+    "gortex-impact",
+    "gortex-overview",
+    "gortex-usages",
+    "gortex-augment",
+]
+
+
+def _make_cache_key(repo_name: str, commit_hash: str) -> str:
+    """Build a deterministic cache directory name from repo and commit."""
+    safe_repo = repo_name.replace("/", "__")
+    return f"{safe_repo}_{commit_hash}"
+
+
+class GortexDockerEnvironment:
+    """Docker environment managing the full container lifecycle for Gortex eval.
+
+    Lifecycle: setup() → (agent runs) → extract_patch() → teardown()
+
+    On any setup failure (container launch, binary copy, health-check timeout),
+    the environment records the failure and returns a result dict with
+    ``exit_status="setup_failure"`` instead of raising.
+    """
+
+    def __init__(
+        self,
+        *,
+        image: str,
+        repo_path: str = CONTAINER_WORKDIR,
+        enable_gortex: bool = True,
+        gortex_binary: str | Path | None = None,
+        gortex_timeout: int = DEFAULT_GORTEX_TIMEOUT,
+        eval_server_port: int = DEFAULT_EVAL_SERVER_PORT,
+        cache_dir: str | Path | None = None,
+        instance_id: str = "",
+    ) -> None:
+        self.image = image
+        self.repo_path = repo_path
+        self.enable_gortex = enable_gortex
+        self.gortex_binary = Path(gortex_binary) if gortex_binary else None
+        self.gortex_timeout = gortex_timeout
+        self.eval_server_port = eval_server_port
+        self.cache_dir = Path(cache_dir) if cache_dir else DEFAULT_CACHE_DIR
+        self.instance_id = instance_id
+
+        self._client: docker.DockerClient | None = None
+        self._container: Any = None  # docker.models.containers.Container
+        self._gortex_ready = False
+        self._setup_error: str | None = None
+        self.index_time: float = 0.0
+
+    # -- public API ----------------------------------------------------------
+
+    def setup(self) -> dict[str, Any] | None:
+        """Launch container, copy gortex + bridge scripts, start eval-server.
+
+        Returns ``None`` on success, or a failure result dict on error.
+        The caller should check the return value and skip agent execution
+        when a failure dict is returned.
+        """
+        try:
+            self._launch_container()
+        except Exception as exc:
+            return self._record_failure(f"Container launch failed: {exc}")
+
+        if not self.enable_gortex:
+            logger.info("Gortex disabled for this instance, skipping tool setup")
+            return None
+
+        try:
+            start = time.time()
+            self._copy_gortex_binary()
+            self._copy_bridge_scripts()
+            self._restore_or_skip_cache()
+            self._start_eval_server()
+            self._wait_for_health()
+            self.index_time = time.time() - start
+            self._gortex_ready = True
+            logger.info(
+                "Gortex environment ready for %s in %.1fs",
+                self.instance_id,
+                self.index_time,
+            )
+        except _SetupTimeout as exc:
+            return self._record_failure(str(exc))
+        except Exception as exc:
+            return self._record_failure(f"Gortex setup failed: {exc}")
+
+        return None
+
+    def extract_patch(self) -> str:
+        """Extract the agent's patch via ``git diff`` inside the container.
+
+        Returns the diff string, or an empty string on failure.
+        """
+        if self._container is None:
+            return ""
+        try:
+            exit_code, output = self._container.exec_run(
+                ["git", "diff"],
+                workdir=self.repo_path,
+            )
+            if exit_code == 0:
+                return output.decode("utf-8", errors="replace")
+            logger.warning("git diff exited %d: %s", exit_code, output[:500])
+        except Exception as exc:
+            logger.warning("Patch extraction failed: %s", exc)
+        return ""
+
+    def teardown(self) -> None:
+        """Stop and remove the container."""
+        if self._container is not None:
+            try:
+                self._container.stop(timeout=5)
+            except Exception:
+                pass
+            try:
+                self._container.remove(force=True)
+            except Exception:
+                pass
+            self._container = None
+        if self._client is not None:
+            try:
+                self._client.close()
+            except Exception:
+                pass
+            self._client = None
+
+    def exec_run(self, cmd: str | list[str], **kwargs: Any) -> tuple[int, str]:
+        """Execute a command inside the container.
+
+        Returns (exit_code, output_string).
+        """
+        if self._container is None:
+            return (1, "Container not running")
+        if isinstance(cmd, str):
+            cmd = ["bash", "-c", cmd]
+        exit_code, output = self._container.exec_run(cmd, **kwargs)
+        return exit_code, output.decode("utf-8", errors="replace")
+
+    @property
+    def is_ready(self) -> bool:
+        """Whether the environment is fully set up and gortex is healthy."""
+        return self._container is not None and (
+            not self.enable_gortex or self._gortex_ready
+        )
+
+    @property
+    def setup_error(self) -> str | None:
+        """Description of setup failure, if any."""
+        return self._setup_error
+
+    # -- private helpers -----------------------------------------------------
+
+    def _launch_container(self) -> None:
+        """Create and start a Docker container from the configured image."""
+        self._client = docker.from_env()
+        logger.info("Launching container from image %s", self.image)
+        self._container = self._client.containers.run(
+            self.image,
+            command="sleep infinity",
+            detach=True,
+            working_dir=self.repo_path,
+        )
+        logger.info("Container %s started", self._container.short_id)
+
+    def _copy_gortex_binary(self) -> None:
+        """Copy the gortex binary into the container."""
+        if self.gortex_binary is None:
+            logger.info("No gortex binary path specified, assuming pre-installed")
+            return
+        if not self.gortex_binary.is_file():
+            raise FileNotFoundError(f"Gortex binary not found: {self.gortex_binary}")
+
+        logger.info("Copying gortex binary into container")
+        self._put_file_in_container(
+            self.gortex_binary,
+            GORTEX_BINARY_CONTAINER_PATH,
+            executable=True,
+        )
+        # Verify binary is accessible and executable
+        exit_code, output = self._container.exec_run(
+            ["ls", "-la", GORTEX_BINARY_CONTAINER_PATH]
+        )
+        if exit_code != 0:
+            raise RuntimeError(
+                f"Gortex binary not found in container after copy: {output.decode()}"
+            )
+        # Check if binary can actually run (catches arch mismatch, missing libs)
+        exit_code, output = self._container.exec_run(
+            [GORTEX_BINARY_CONTAINER_PATH, "--help"]
+        )
+        if exit_code != 0:
+            # Try to get more info
+            _, file_output = self._container.exec_run(
+                ["file", GORTEX_BINARY_CONTAINER_PATH]
+            )
+            _, ldd_output = self._container.exec_run(
+                ["bash", "-c", f"ldd {GORTEX_BINARY_CONTAINER_PATH} 2>&1 || true"]
+            )
+            raise RuntimeError(
+                f"Gortex binary cannot execute in container.\n"
+                f"  file: {file_output.decode().strip()}\n"
+                f"  ldd: {ldd_output.decode().strip()}\n"
+                f"  error: {output.decode().strip()}"
+            )
+        logger.info("Gortex binary verified at %s", GORTEX_BINARY_CONTAINER_PATH)
+
+    def _copy_bridge_scripts(self) -> None:
+        """Copy tool bridge bash scripts into /usr/local/bin/ in the container."""
+        bridge_dir = Path(__file__).resolve().parent.parent / "bridge"
+        copied = 0
+        for script_name in _BRIDGE_SCRIPTS:
+            script_path = bridge_dir / script_name
+            if not script_path.is_file():
+                logger.warning("Bridge script not found: %s", script_path)
+                continue
+            self._put_file_in_container(
+                script_path,
+                f"{BRIDGE_SCRIPTS_CONTAINER_DIR}/{script_name}",
+                executable=True,
+            )
+            copied += 1
+        logger.info("Copied %d bridge scripts into container", copied)
+
+    def _restore_or_skip_cache(self) -> None:
+        """Mount/copy a cached index into the container if one exists."""
+        repo_name, commit_hash = self._get_repo_identity()
+        cache_key = _make_cache_key(repo_name, commit_hash)
+        cache_path = self.cache_dir / cache_key
+
+        if not cache_path.is_dir():
+            logger.info("No cached index for %s, eval-server will index fresh", cache_key)
+            return
+
+        tarball = cache_path / "index.tar.gz"
+        if not tarball.is_file():
+            logger.info("Cache dir exists but no tarball for %s, skipping", cache_key)
+            return
+
+        logger.info("Restoring cached index %s into container", cache_key)
+        try:
+            cache_dest = "/root/.gortex-cache"
+            self._container.exec_run(["mkdir", "-p", cache_dest])
+            with open(tarball, "rb") as f:
+                self._container.put_archive(cache_dest, f.read())
+            logger.info("Cached index restored to %s", cache_dest)
+        except Exception as exc:
+            logger.warning("Cache restore failed, will index fresh: %s", exc)
+
+    def _start_eval_server(self) -> None:
+        """Start ``gortex eval-server`` as a background process in the container."""
+        cache_flag = ""
+        cache_dest = "/root/.gortex-cache"
+        # Check if cache was restored
+        exit_code, _ = self._container.exec_run(["test", "-d", cache_dest])
+        if exit_code == 0:
+            cache_flag = f"--cache-dir {cache_dest}"
+
+        cmd = (
+            f"nohup /usr/local/bin/gortex eval-server "
+            f"--port {self.eval_server_port} "
+            f"--index {self.repo_path} "
+            f"{cache_flag} "
+            f"> /tmp/gortex-eval-server.log 2>&1 &"
+        )
+        logger.info("Starting eval-server on port %d", self.eval_server_port)
+        self._container.exec_run(["bash", "-c", cmd], detach=True)
+
+    def _wait_for_health(self) -> None:
+        """Poll the eval-server health endpoint until ready or timeout.
+
+        Raises ``_SetupTimeout`` if the server doesn't become healthy
+        within ``self.gortex_timeout`` seconds.
+        """
+        deadline = time.time() + self.gortex_timeout
+        health_url = f"http://127.0.0.1:{self.eval_server_port}/health"
+        attempt = 0
+
+        while time.time() < deadline:
+            time.sleep(HEALTH_CHECK_INTERVAL)
+            attempt += 1
+            try:
+                exit_code, output = self._container.exec_run(
+                    ["curl", "-sf", health_url],
+                )
+                if exit_code == 0 and b'"status"' in output and b'"ok"' in output:
+                    elapsed = self.gortex_timeout - (deadline - time.time())
+                    logger.info(
+                        "Eval-server healthy after %d attempts (%.1fs)",
+                        attempt,
+                        elapsed,
+                    )
+                    return
+            except Exception:
+                pass
+
+        # Grab server logs for diagnostics
+        try:
+            _, log_output = self._container.exec_run(
+                ["tail", "-30", "/tmp/gortex-eval-server.log"],
+            )
+            log_tail = log_output.decode("utf-8", errors="replace")[-1000:]
+        except Exception:
+            log_tail = "(unavailable)"
+
+        raise _SetupTimeout(
+            f"Eval-server health check timed out after {self.gortex_timeout}s "
+            f"for instance {self.instance_id}. Server log tail:\n{log_tail}"
+        )
+
+    def _get_repo_identity(self) -> tuple[str, str]:
+        """Extract (repo_name, commit_hash) from the container's /testbed repo."""
+        _, repo_out = self._container.exec_run(
+            ["bash", "-c", "basename $(git remote get-url origin 2>/dev/null || basename $(pwd)) .git"],
+            workdir=self.repo_path,
+        )
+        _, commit_out = self._container.exec_run(
+            ["bash", "-c", "git rev-parse HEAD 2>/dev/null || echo unknown"],
+            workdir=self.repo_path,
+        )
+        repo_name = repo_out.decode("utf-8", errors="replace").strip() or "unknown"
+        commit_hash = commit_out.decode("utf-8", errors="replace").strip() or "unknown"
+        return repo_name, commit_hash
+
+    def _put_file_in_container(
+        self,
+        local_path: Path,
+        container_path: str,
+        *,
+        executable: bool = False,
+    ) -> None:
+        """Copy a local file into the container using the Docker API.
+
+        Falls back to ``podman cp`` / ``docker cp`` if the API method fails.
+        """
+        dest_dir = str(Path(container_path).parent)
+
+        # Try Docker SDK put_archive first
+        try:
+            data = local_path.read_bytes()
+            tar_stream = io.BytesIO()
+            with tarfile.open(fileobj=tar_stream, mode="w") as tar:
+                info = tarfile.TarInfo(name=Path(container_path).name)
+                info.size = len(data)
+                if executable:
+                    info.mode = 0o755
+                tar.addfile(info, io.BytesIO(data))
+            tar_stream.seek(0)
+            self._container.put_archive(dest_dir, tar_stream)
+
+            # Verify the file actually landed
+            exit_code, _ = self._container.exec_run(["test", "-f", container_path])
+            if exit_code == 0:
+                if executable:
+                    self._container.exec_run(["chmod", "+x", container_path])
+                return
+            logger.warning("put_archive succeeded but file not found, trying cp fallback")
+        except Exception as exc:
+            logger.warning("put_archive failed (%s), trying cp fallback", exc)
+
+        # Fallback: use podman/docker cp via subprocess
+        import subprocess
+        container_id = self._container.short_id
+        try:
+            subprocess.run(
+                ["podman", "cp", str(local_path), f"{container_id}:{container_path}"],
+                check=True, capture_output=True, timeout=30,
+            )
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            # Try docker cp as last resort
+            subprocess.run(
+                ["docker", "cp", str(local_path), f"{container_id}:{container_path}"],
+                check=True, capture_output=True, timeout=30,
+            )
+        if executable:
+            self._container.exec_run(["chmod", "+x", container_path])
+
+    def _record_failure(self, message: str) -> dict[str, Any]:
+        """Record a setup failure and return a result dict for the runner."""
+        logger.error("Setup failure for %s: %s", self.instance_id, message)
+        self._setup_error = message
+        return {
+            "instance_id": self.instance_id,
+            "exit_status": "setup_failure",
+            "setup_error": message,
+            "submission": "",
+        }
+
+
+class _SetupTimeout(Exception):
+    """Raised internally when the eval-server health check times out."""

eval/prompts.py

@@ -0,0 +1,82 @@
+"""Prompt template loader for the eval framework.
+
+Loads Jinja2 system and instance prompt templates per evaluation mode
+from the ``eval/prompts/`` directory.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Tuple
+
+import jinja2
+
+# Valid evaluation modes
+VALID_MODES = ("baseline", "native", "native_augment")
+
+# Prompts directory lives alongside this module
+_PROMPTS_DIR = Path(__file__).resolve().parent / "prompts"
+
+
+def _make_env() -> jinja2.Environment:
+    """Create a Jinja2 environment rooted at the prompts directory."""
+    return jinja2.Environment(
+        loader=jinja2.FileSystemLoader(str(_PROMPTS_DIR)),
+        keep_trailing_newline=True,
+    )
+
+
+def load_templates(mode: str) -> Tuple[jinja2.Template, jinja2.Template]:
+    """Load system and instance Jinja2 templates for *mode*.
+
+    Parameters
+    ----------
+    mode:
+        One of ``baseline``, ``native``, or ``native_augment``.
+
+    Returns
+    -------
+    tuple[jinja2.Template, jinja2.Template]
+        ``(system_template, instance_template)``
+
+    Raises
+    ------
+    FileNotFoundError
+        If either template file does not exist for the given mode.
+    """
+    env = _make_env()
+
+    system_name = f"system_{mode}.jinja"
+    instance_name = f"instance_{mode}.jinja"
+
+    system_path = _PROMPTS_DIR / system_name
+    instance_path = _PROMPTS_DIR / instance_name
+
+    if not system_path.is_file():
+        raise FileNotFoundError(f"System template not found: {system_path}")
+    if not instance_path.is_file():
+        raise FileNotFoundError(f"Instance template not found: {instance_path}")
+
+    system_template = env.get_template(system_name)
+    instance_template = env.get_template(instance_name)
+
+    return system_template, instance_template
+
+
+def render_instance_prompt(template: jinja2.Template, task: str) -> str:
+    """Render an instance prompt template with the given *task*.
+
+    Parameters
+    ----------
+    template:
+        A Jinja2 ``Template`` object (typically the instance template
+        returned by :func:`load_templates`).
+    task:
+        The SWE-bench issue description to inject into the template.
+
+    Returns
+    -------
+    str
+        The fully rendered prompt string.
+    """
+    return template.render(task=task)

eval/prompts/.gitkeep

@@ -0,0 +1 @@
+

eval/prompts/instance_baseline.jinja

@@ -0,0 +1,81 @@
+Please solve this issue: {{task}}
+
+You can execute bash commands and edit files to implement the necessary changes.
+
+## Recommended Workflow
+
+This workflows should be done step-by-step so that you can iterate on your changes and any possible problems.
+
+1. Analyze the codebase by finding and reading relevant files
+2. Create a script to reproduce the issue
+3. Edit the source code to resolve the issue
+4. Verify your fix works by running your script again
+5. Test edge cases to ensure your fix is robust
+6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.
+   Do not combine it with any other command. After this command, you cannot continue working on this task.
+
+## Important Rules
+
+1. Every response must contain exactly one action
+2. The action must be enclosed in triple backticks
+3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell.
+   However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files
+4. Make minimal, targeted changes. Don't refactor unrelated code.
+
+<system_info>
+{{system}} {{release}} {{version}} {{machine}}
+</system_info>
+
+## Formatting your response
+
+Here is an example of a correct response:
+
+<example_response>
+THOUGHT: I need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.
+
+```mswea_bash_command
+ls -la
+```
+</example_response>
+
+## Useful command examples
+
+### Create a new file:
+
+```bash
+cat <<'EOF' > newfile.py
+import numpy as np
+hello = "world"
+print(hello)
+EOF
+```
+
+### Edit files with sed:
+
+{%- if system == "Darwin" -%}
+<note>
+You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.
+</note>
+{%- endif -%}
+
+```bash
+# Replace all occurrences
+sed -i 's/old_string/new_string/g' filename.py
+# Replace only first occurrence
+sed -i 's/old_string/new_string/' filename.py
+# Replace all occurrences in lines 1-10
+sed -i '1,10s/old_string/new_string/g' filename.py
+```
+
+### View file content:
+
+```bash
+# View specific lines with numbers
+nl -ba filename.py | sed -n '10,20p'
+```
+
+### Any other command you want to run
+
+```bash
+anything
+```

eval/prompts/instance_native.jinja

@@ -0,0 +1,102 @@
+Please solve this issue: {{task}}
+
+You can execute bash commands and edit files to implement the necessary changes.
+
+## Recommended Workflow
+
+Work step-by-step so you can iterate on your changes and catch problems early.
+
+1. **Understand the issue** — read the problem statement, identify the symptom and affected area
+2. **Find the relevant code** — use `gortex-search "<concept>"` to find symbols, or `grep` for specific strings
+3. **Understand the suspect** — use `gortex-context "<task description>"` to get full context with callers, callees, and execution flows
+4. **Check blast radius** — before editing shared code, run `gortex-impact "<symbol_id>"` to see what depends on it
+5. **Implement the fix** — make minimal, targeted changes
+6. **Verify** — run relevant tests, check edge cases
+7. **Submit** — issue: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`
+   Do not combine it with any other command. After this command, you cannot continue working on this task.
+
+## Debugging Patterns
+
+| Symptom | Approach |
+|---------|----------|
+| Error message / exception | `gortex-search` for error text → `gortex-context` on the affected area |
+| Wrong return value | `gortex-context` on the function → trace callees for data flow |
+| Missing feature / incomplete behavior | `gortex-search` for feature area → `gortex-context` to find the gap |
+| Need to understand callers | `gortex-usages` — graph-complete, finds callers grep would miss |
+
+## Risk Assessment
+
+Before editing shared code, check the blast radius with `gortex-impact`:
+
+| Impact | Risk | Action |
+|--------|------|--------|
+| <5 symbols at d=1 | Low | Fix with confidence |
+| 5-15 symbols at d=1 | Medium | Fix carefully, run broader tests |
+| >15 symbols at d=1 | High | Minimal change, run full test suite |
+
+## Important Rules
+
+1. Every response must contain exactly one action
+2. The action must be enclosed in triple backticks
+3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell.
+   However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files
+4. Make minimal, targeted changes. Don't refactor unrelated code.
+5. Gortex tools are ~100ms. Use them when they save you multiple grep iterations.
+
+<system_info>
+{{system}} {{release}} {{version}} {{machine}}
+</system_info>
+
+## Formatting your response
+
+Here is an example of a correct response:
+
+<example_response>
+THOUGHT: The issue mentions a problem with form field validation. Let me search the code knowledge graph for relevant symbols to understand how validation works in this codebase.
+
+```mswea_bash_command
+gortex-search "form field validation"
+```
+</example_response>
+
+## Useful command examples
+
+### Create a new file:
+
+```bash
+cat <<'EOF' > newfile.py
+import numpy as np
+hello = "world"
+print(hello)
+EOF
+```
+
+### Edit files with sed:
+
+{%- if system == "Darwin" -%}
+<note>
+You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.
+</note>
+{%- endif -%}
+
+```bash
+# Replace all occurrences
+sed -i 's/old_string/new_string/g' filename.py
+# Replace only first occurrence
+sed -i 's/old_string/new_string/' filename.py
+# Replace all occurrences in lines 1-10
+sed -i '1,10s/old_string/new_string/g' filename.py
+```
+
+### View file content:
+
+```bash
+# View specific lines with numbers
+nl -ba filename.py | sed -n '10,20p'
+```
+
+### Any other command you want to run
+
+```bash
+anything
+```

eval/prompts/instance_native_augment.jinja

@@ -0,0 +1,119 @@
+Please solve this issue: {{task}}
+
+You can execute bash commands and edit files to implement the necessary changes.
+
+## Recommended Workflow
+
+Work step-by-step so you can iterate on your changes and catch problems early.
+
+1. **Understand the issue** — read the problem statement, identify the symptom and affected area
+2. **Find the relevant code** — use `gortex-search "<concept>"` to find symbols, or `grep` for specific strings
+3. **Understand the suspect** — use `gortex-context "<task description>"` to get full context with callers, callees, and execution flows
+4. **Check blast radius** — before editing shared code, run `gortex-impact "<symbol_id>"` to see what depends on it
+5. **Implement the fix** — make minimal, targeted changes
+6. **Verify** — run relevant tests, check edge cases
+7. **Submit** — issue: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`
+   Do not combine it with any other command. After this command, you cannot continue working on this task.
+
+## Using [Gortex] Annotations
+
+When you run `grep` or `rg`, the output is automatically enriched with `[Gortex]` annotations:
+
+```
+src/auth/service.py:42:def validate_token(token):
+  [Gortex] callers: handle_request (api/handler.py:15), auth_middleware (middleware/auth.py:8)
+  [Gortex] callees: parse_jwt (crypto/jwt.py:22), check_expiry (auth/expiry.py:5)
+```
+
+**How to use annotations:**
+- **callers** tell you what depends on this symbol — check these before making changes
+- **callees** tell you what this symbol uses — follow these to trace data flow
+- Use annotations to navigate directly instead of running extra search commands
+- If annotations show many callers, run `gortex-impact` to get the full blast radius
+
+## Debugging Patterns
+
+| Symptom | Approach |
+|---------|----------|
+| Error message / exception | `grep` for error text (check `[Gortex]` annotations) → `gortex-context` on the affected area |
+| Wrong return value | `grep` for the function (annotations show callees) → trace data flow |
+| Missing feature / incomplete behavior | `gortex-search` for feature area → `gortex-context` to find the gap |
+| Need to understand callers | Check `[Gortex]` annotations first, then `gortex-usages` for the complete picture |
+
+## Risk Assessment
+
+Before editing shared code, check the blast radius with `gortex-impact`:
+
+| Impact | Risk | Action |
+|--------|------|--------|
+| <5 symbols at d=1 | Low | Fix with confidence |
+| 5-15 symbols at d=1 | Medium | Fix carefully, run broader tests |
+| >15 symbols at d=1 | High | Minimal change, run full test suite |
+
+## Important Rules
+
+1. Every response must contain exactly one action
+2. The action must be enclosed in triple backticks
+3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell.
+   However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files
+4. Make minimal, targeted changes. Don't refactor unrelated code.
+5. Gortex tools are ~100ms. Use them when they save you multiple grep iterations.
+6. When grep results show `[Gortex]` enrichments, use those for navigation.
+
+<system_info>
+{{system}} {{release}} {{version}} {{machine}}
+</system_info>
+
+## Formatting your response
+
+Here is an example of a correct response:
+
+<example_response>
+THOUGHT: The issue mentions a problem with form field validation. Let me search the code knowledge graph for relevant symbols to understand how validation works in this codebase.
+
+```mswea_bash_command
+gortex-search "form field validation"
+```
+</example_response>
+
+## Useful command examples
+
+### Create a new file:
+
+```bash
+cat <<'EOF' > newfile.py
+import numpy as np
+hello = "world"
+print(hello)
+EOF
+```
+
+### Edit files with sed:
+
+{%- if system == "Darwin" -%}
+<note>
+You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.
+</note>
+{%- endif -%}
+
+```bash
+# Replace all occurrences
+sed -i 's/old_string/new_string/g' filename.py
+# Replace only first occurrence
+sed -i 's/old_string/new_string/' filename.py
+# Replace all occurrences in lines 1-10
+sed -i '1,10s/old_string/new_string/g' filename.py
+```
+
+### View file content:
+
+```bash
+# View specific lines with numbers
+nl -ba filename.py | sed -n '10,20p'
+```
+
+### Any other command you want to run
+
+```bash
+anything
+```

eval/prompts/system_baseline.jinja

@@ -0,0 +1,15 @@
+You are a helpful assistant that can interact with a computer to solve software engineering tasks.
+
+Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||).
+Include a THOUGHT section before your command where you explain your reasoning process.
+Format your response as shown in.
+
+<example_response>
+Your reasoning and analysis here. Explain why you want to perform the action.
+
+```mswea_bash_command
+your_command_here
+```
+</example_response>
+
+Failure to follow these rules will cause your response to be rejected.

eval/prompts/system_native.jinja

@@ -0,0 +1,63 @@
+You are a helpful assistant that can interact with a computer to solve software engineering tasks.
+
+Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||).
+Include a THOUGHT section before your command where you explain your reasoning process.
+Format your response as shown in.
+
+<example_response>
+Your reasoning and analysis here. Explain why you want to perform the action.
+
+```mswea_bash_command
+your_command_here
+```
+</example_response>
+
+Failure to follow these rules will cause your response to be rejected.
+
+## Code Intelligence
+
+You have **Gortex** — a code intelligence engine that indexes this codebase into an in-memory knowledge graph. It knows every function call chain, class hierarchy, execution flow, and symbol relationship. These are fast bash commands (~100ms). Use them when useful, skip them when a simple grep suffices.
+
+### Gortex Commands
+
+**gortex-search "<query>"** — BM25 symbol search with camelCase-aware tokenization.
+Finds functions, classes, types, and variables by name or concept. Returns ranked results with file locations.
+```bash
+gortex-search "validate token auth"
+```
+
+**gortex-context "<task_description>"** — 360-degree smart context for a task.
+Returns relevant symbols, their source, callers, callees, execution flows, and an edit plan. Graph-complete — finds relationships that grep misses.
+```bash
+gortex-context "fix form field validation in BoundField"
+```
+
+**gortex-impact "<symbol_id>"** — Blast radius analysis.
+What breaks if you change this: d=1 WILL BREAK, d=2 LIKELY AFFECTED, d=3 MAY NEED TESTING.
+```bash
+gortex-impact "django/forms/boundfield.py::BoundField"
+```
+
+**gortex-overview** — Graph statistics and codebase overview.
+Returns node/edge counts, language breakdown, and index health.
+```bash
+gortex-overview
+```
+
+**gortex-usages "<symbol_id>"** — Find all references to a symbol across the codebase.
+Zero false positives — uses the graph, not text matching.
+```bash
+gortex-usages "django/forms/fields.py::CharField"
+```
+
+### When to Use What
+
+| I need to... | Use |
+|---|---|
+| Find a function/class by name or concept | `gortex-search` |
+| Understand how a feature works end-to-end | `gortex-context` |
+| Find ALL callers/usages of a symbol | `gortex-usages` |
+| Know what breaks if I change something | `gortex-impact` |
+| Get a codebase overview | `gortex-overview` |
+| Find a string literal or error message | `grep` |
+| Read source code | `cat` / `nl -ba` |

eval/prompts/system_native_augment.jinja

@@ -0,0 +1,81 @@
+You are a helpful assistant that can interact with a computer to solve software engineering tasks.
+
+Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||).
+Include a THOUGHT section before your command where you explain your reasoning process.
+Format your response as shown in.
+
+<example_response>
+Your reasoning and analysis here. Explain why you want to perform the action.
+
+```mswea_bash_command
+your_command_here
+```
+</example_response>
+
+Failure to follow these rules will cause your response to be rejected.
+
+## Code Intelligence
+
+You have **Gortex** — a code intelligence engine that indexes this codebase into an in-memory knowledge graph. It knows every function call chain, class hierarchy, execution flow, and symbol relationship. These are fast bash commands (~100ms). Use them when useful, skip them when a simple grep suffices.
+
+Your `grep` and `rg` results are also automatically enriched with `[Gortex]` annotations showing callers, callees, and execution flows for matched symbols. Pay attention to these — they often point you to the right code without extra tool calls.
+
+### Gortex Commands
+
+**gortex-search "<query>"** — BM25 symbol search with camelCase-aware tokenization.
+Finds functions, classes, types, and variables by name or concept. Returns ranked results with file locations.
+```bash
+gortex-search "validate token auth"
+```
+
+**gortex-context "<task_description>"** — 360-degree smart context for a task.
+Returns relevant symbols, their source, callers, callees, execution flows, and an edit plan. Graph-complete — finds relationships that grep misses.
+```bash
+gortex-context "fix form field validation in BoundField"
+```
+
+**gortex-impact "<symbol_id>"** — Blast radius analysis.
+What breaks if you change this: d=1 WILL BREAK, d=2 LIKELY AFFECTED, d=3 MAY NEED TESTING.
+```bash
+gortex-impact "django/forms/boundfield.py::BoundField"
+```
+
+**gortex-overview** — Graph statistics and codebase overview.
+Returns node/edge counts, language breakdown, and index health.
+```bash
+gortex-overview
+```
+
+**gortex-usages "<symbol_id>"** — Find all references to a symbol across the codebase.
+Zero false positives — uses the graph, not text matching.
+```bash
+gortex-usages "django/forms/fields.py::CharField"
+```
+
+### Interpreting [Gortex] Annotations
+
+When you run `grep` or `rg`, the output may include `[Gortex]` annotations like:
+
+```
+src/auth/service.py:42:def validate_token(token):
+  [Gortex] callers: handle_request (api/handler.py:15), auth_middleware (middleware/auth.py:8)
+  [Gortex] callees: parse_jwt (crypto/jwt.py:22), check_expiry (auth/expiry.py:5)
+```
+
+These annotations tell you:
+- **callers** — which functions call this symbol (useful for understanding impact)
+- **callees** — which functions this symbol calls (useful for tracing data flow)
+
+Use annotations to navigate the codebase efficiently without extra tool calls.
+
+### When to Use What
+
+| I need to... | Use |
+|---|---|
+| Find a function/class by name or concept | `gortex-search` |
+| Understand how a feature works end-to-end | `gortex-context` |
+| Find ALL callers/usages of a symbol | `gortex-usages` |
+| Know what breaks if I change something | `gortex-impact` |
+| Get a codebase overview | `gortex-overview` |
+| Find a string literal or error message | `grep` (with `[Gortex]` enrichment) |
+| Read source code | `cat` / `nl -ba` |

eval/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "gortex-eval"
+version = "0.1.0"
+description = "SWE-bench evaluation harness for Gortex code intelligence"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "litellm>=1.50.0",
+    "docker>=7.0.0",
+    "pyyaml>=6.0",
+    "jinja2>=3.1.0",
+    "datasets>=3.0.0",
+    "tabulate>=0.9.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "hypothesis>=6.88.0",
+    "pytest>=8.0.0",
+]
+
+[project.scripts]
+gortex-eval = "run_eval:main"
+gortex-eval-analyze = "analysis.analyze_results:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"

eval/results.py

@@ -0,0 +1,160 @@
+"""Result store for the Gortex eval framework.
+
+Provides dataclasses for per-instance results and run summaries, plus
+persistence helpers that write JSON to the results directory.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class InstanceResult:
+    """Per-instance evaluation result with all metric fields."""
+
+    instance_id: str
+    model: str
+    mode: str
+    exit_status: str
+    submission: str = ""
+    cost: float = 0.0
+    tokens_input: int = 0
+    tokens_output: int = 0
+    n_calls: int = 0
+    n_steps: int = 0
+    duration_seconds: float = 0.0
+    gortex_metrics: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict."""
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> InstanceResult:
+        """Deserialize from a plain dict."""
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class RunSummary:
+    """Aggregate summary for an evaluation run."""
+
+    run_id: str
+    model: str
+    mode: str
+    timestamp: float = 0.0
+    config: dict[str, Any] = field(default_factory=dict)
+    total_instances: int = 0
+    completed: int = 0
+    patch_rate: float = 0.0
+    total_cost: float = 0.0
+    mean_cost: float = 0.0
+    total_tokens: int = 0
+    mean_tokens: float = 0.0
+    total_duration_seconds: float = 0.0
+    mean_duration_seconds: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dict."""
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunSummary:
+        """Deserialize from a plain dict."""
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+
+def save_instance_result(result: InstanceResult, run_id: str, base_dir: Path | None = None) -> Path:
+    """Write an instance result to ``results/{run_id}/{instance_id}/{instance_id}.json``.
+
+    Returns the path to the written file.
+    """
+    base = base_dir or Path("results")
+    out_dir = base / run_id / result.instance_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / f"{result.instance_id}.json"
+    out_path.write_text(json.dumps(result.to_dict(), indent=2))
+    return out_path
+
+
+def save_run_summary(
+    results: list[InstanceResult],
+    run_id: str,
+    config: dict[str, Any],
+    base_dir: Path | None = None,
+) -> RunSummary:
+    """Compute aggregate metrics from *results* and write ``results/{run_id}/summary.json``.
+
+    Returns the computed :class:`RunSummary`.
+    """
+    base = base_dir or Path("results")
+    out_dir = base / run_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    total = len(results)
+    if total == 0:
+        summary = RunSummary(
+            run_id=run_id,
+            model="",
+            mode="",
+            timestamp=time.time(),
+            config=config,
+        )
+    else:
+        patches = sum(1 for r in results if r.submission)
+        total_cost = sum(r.cost for r in results)
+        total_tokens = sum(r.tokens_input + r.tokens_output for r in results)
+        total_duration = sum(r.duration_seconds for r in results)
+        completed = sum(1 for r in results if r.exit_status == "submitted")
+
+        summary = RunSummary(
+            run_id=run_id,
+            model=results[0].model,
+            mode=results[0].mode,
+            timestamp=time.time(),
+            config=config,
+            total_instances=total,
+            completed=completed,
+            patch_rate=patches / total,
+            total_cost=total_cost,
+            mean_cost=total_cost / total,
+            total_tokens=total_tokens,
+            mean_tokens=total_tokens / total,
+            total_duration_seconds=total_duration,
+            mean_duration_seconds=total_duration / total,
+        )
+
+    out_path = out_dir / "summary.json"
+    out_path.write_text(json.dumps(summary.to_dict(), indent=2))
+    return summary
+
+
+def save_predictions(
+    results: list[InstanceResult],
+    run_id: str,
+    base_dir: Path | None = None,
+) -> Path:
+    """Write SWE-bench compatible ``preds.json`` to ``results/{run_id}/preds.json``.
+
+    Returns the path to the written file.
+    """
+    base = base_dir or Path("results")
+    out_dir = base / run_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    preds: dict[str, dict[str, str]] = {}
+    for r in results:
+        preds[r.instance_id] = {
+            "model_name_or_path": r.model,
+            "instance_id": r.instance_id,
+            "model_patch": r.submission,
+        }
+
+    out_path = out_dir / "preds.json"
+    out_path.write_text(json.dumps(preds, indent=2))
+    return out_path

eval/run_eval.py

@@ -0,0 +1,797 @@
+"""Eval runner CLI for Gortex SWE-bench evaluation.
+
+Orchestrates evaluation runs across models and modes:
+- ``single`` — run one (model, mode) configuration
+- ``matrix`` — run full cross-product of models × modes
+- ``debug`` — run a single instance with verbose logging
+- ``list-configs`` — show available model/mode configs
+
+Entry point: ``main()`` (referenced in pyproject.toml as ``gortex-eval``).
+
+Heavy lifting lives elsewhere:
+- Config: ``eval.config`` (load, merge, validate YAML configs)
+- Environment: ``eval.environments.gortex_docker`` (container lifecycle)
+- Agent: ``eval.agents.gortex_agent`` (LLM agent wrapper)
+- Metrics persistence: ``eval.results`` (task 15, not yet implemented)
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import logging
+import sys
+import time
+from itertools import product
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger("gortex_eval")
+
+# ---------------------------------------------------------------------------
+# Dataset name mapping
+# ---------------------------------------------------------------------------
+
+DATASET_MAPPING: Dict[str, str] = {
+    "lite": "princeton-nlp/SWE-bench_Lite",
+    "verified": "princeton-nlp/SWE-bench_Verified",
+    "full": "princeton-nlp/SWE-bench",
+}
+
+DEFAULT_OUTPUT_DIR = Path("results")
+DEFAULT_SUBSET = "lite"
+DEFAULT_SPLIT = "test"
+
+
+# ---------------------------------------------------------------------------
+# Pure helper functions (exported for testing)
+# ---------------------------------------------------------------------------
+
+def parse_slice(spec: str) -> slice:
+    """Parse a slice spec string into a Python ``slice`` object.
+
+    Supports the same semantics as Python's ``list[start:end]``:
+    - ``"0:5"``  → ``slice(0, 5)``
+    - ``"10:20"`` → ``slice(10, 20)``
+    - ``":3"``   → ``slice(None, 3)``
+    - ``"5:"``   → ``slice(5, None)``
+    - ``"::2"``  → ``slice(None, None, 2)``
+    - ``""``     → ``slice(None)`` (no-op, selects everything)
+
+    Raises ``ValueError`` for malformed specs.
+    """
+    spec = spec.strip()
+    if not spec:
+        return slice(None)
+
+    parts = spec.split(":")
+    if len(parts) > 3:
+        raise ValueError(f"Invalid slice spec: {spec!r} (too many colons)")
+
+    def _parse_part(s: str) -> Optional[int]:
+        s = s.strip()
+        if not s:
+            return None
+        try:
+            return int(s)
+        except ValueError:
+            raise ValueError(f"Invalid slice component: {s!r} in {spec!r}")
+
+    parsed = [_parse_part(p) for p in parts]
+
+    if len(parsed) == 1:
+        # Single value like "5" — treat as "0:5" for convenience
+        return slice(None, parsed[0])
+    elif len(parsed) == 2:
+        return slice(parsed[0], parsed[1])
+    else:
+        return slice(parsed[0], parsed[1], parsed[2])
+
+
+def generate_run_id(model_name: str, mode_name: str, timestamp: Optional[float] = None) -> str:
+    """Generate a unique run ID from model name, mode, and timestamp.
+
+    Format: ``{model}_{mode}_{timestamp}``
+    """
+    ts = int(timestamp or time.time())
+    return f"{model_name}_{mode_name}_{ts}"
+
+
+def build_matrix_configs(
+    models: List[str], modes: List[str]
+) -> List[Tuple[str, str]]:
+    """Build the cross-product of (model, mode) pairs.
+
+    Returns a list of unique ``(model_name, mode_name)`` tuples.
+    """
+    return list(product(models, modes))
+
+
+# ---------------------------------------------------------------------------
+# Instance loading
+# ---------------------------------------------------------------------------
+
+def load_instances(
+    subset: str = DEFAULT_SUBSET,
+    split: str = DEFAULT_SPLIT,
+    slice_spec: str = "",
+    filter_spec: str = "",
+) -> List[Dict[str, Any]]:
+    """Load SWE-bench instances from HuggingFace datasets.
+
+    Parameters
+    ----------
+    subset:
+        Dataset subset name (``lite``, ``verified``, ``full``) or a full
+        HuggingFace dataset path.
+    split:
+        Dataset split (e.g., ``test``, ``dev``).
+    slice_spec:
+        Optional slice spec string (e.g., ``"0:5"``) applied after loading.
+    filter_spec:
+        Optional regex to filter instance IDs.
+
+    Returns
+    -------
+    list[dict]
+        List of SWE-bench instance dicts.
+    """
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        logger.error(
+            "The 'datasets' package is required for loading SWE-bench instances. "
+            "Install it with: pip install datasets"
+        )
+        sys.exit(1)
+
+    import re
+
+    dataset_path = DATASET_MAPPING.get(subset, subset)
+    logger.info("Loading dataset: %s, split: %s", dataset_path, split)
+    instances = list(load_dataset(dataset_path, split=split))
+
+    if filter_spec:
+        pattern = re.compile(filter_spec)
+        instances = [i for i in instances if pattern.match(i["instance_id"])]
+
+    if slice_spec:
+        sl = parse_slice(slice_spec)
+        instances = instances[sl]
+
+    logger.info("Loaded %d instances", len(instances))
+    return instances
+
+
+# ---------------------------------------------------------------------------
+# Run orchestration
+# ---------------------------------------------------------------------------
+
+def _build_config(model_name: str, mode_name: str) -> Dict[str, Any]:
+    """Load and merge model + mode configs, validate the result."""
+    from config import load_model_config, load_mode_config, merge_configs, validate_config
+
+    model_cfg = load_model_config(model_name)
+    mode_cfg = load_mode_config(mode_name)
+    merged = merge_configs(model_cfg, mode_cfg)
+    validate_config(merged)
+    return merged
+
+
+def _get_instance_image(instance: Dict[str, Any], env_cfg: Dict[str, Any]) -> str:
+    """Derive the SWE-bench Docker image name for an instance.
+
+    SWE-bench v4 names images as:
+      sweb.eval.x86_64.{instance_id}:{tag}
+    Falls back to the instance's environment_image_key if present.
+    """
+    tag = env_cfg.get("swebench_image_tag", "sweb.eval.x86_64")
+    instance_id = instance["instance_id"]
+    # Check if instance has an explicit image key
+    image_key = instance.get("environment_image_key", "")
+    if image_key and not image_key.startswith("swebench/"):
+        return image_key
+    # SWE-bench v4 convention
+    return f"sweb.eval.x86_64.{instance_id}:{tag}"
+
+
+def process_instance(
+    instance: Dict[str, Any],
+    config: Dict[str, Any],
+    output_dir: Path,
+    run_id: str,
+    model_name: str,
+    mode_name: str,
+) -> Dict[str, Any]:
+    """Process a single SWE-bench instance.
+
+    Lifecycle: launch container → setup → drive agent → extract patch →
+    collect metrics → teardown.
+
+    On non-recoverable error: log, record as failed, return result dict.
+    """
+    instance_id = instance["instance_id"]
+    instance_dir = output_dir / run_id / instance_id
+    instance_dir.mkdir(parents=True, exist_ok=True)
+
+    agent_cfg = config.get("agent", {})
+    cost_limit = float(agent_cfg.get("cost_limit", 3.0))
+    step_limit = int(agent_cfg.get("step_limit", 30))
+
+    result: Dict[str, Any] = {
+        "instance_id": instance_id,
+        "model": model_name,
+        "mode": mode_name,
+        "exit_status": None,
+        "submission": "",
+        "cost": 0.0,
+        "tokens_input": 0,
+        "tokens_output": 0,
+        "n_calls": 0,
+        "n_steps": 0,
+        "duration_seconds": 0.0,
+        "gortex_metrics": {},
+    }
+
+    env = None
+    agent = None
+    start_time = time.time()
+
+    try:
+        # --- environment setup ---
+        from environments.gortex_docker import GortexDockerEnvironment
+
+        env_cfg = config.get("environment", {})
+        instance_image = _get_instance_image(instance, env_cfg)
+        env = GortexDockerEnvironment(
+            image=instance_image,
+            enable_gortex=env_cfg.get("enable_gortex", True),
+            gortex_binary=env_cfg.get("gortex_binary"),
+            gortex_timeout=int(env_cfg.get("gortex_timeout", 120)),
+            eval_server_port=int(env_cfg.get("eval_server_port", 4747)),
+            cache_dir=env_cfg.get("cache_dir"),
+            instance_id=instance_id,
+        )
+        env.setup()
+
+        # --- agent setup ---
+        from agents.gortex_agent import GortexAgent
+
+        model_cfg = config.get("model", {})
+        model_name_full = model_cfg.get("model_name", model_name)
+
+        # Create mini-swe-agent Model and Environment
+        try:
+            from minisweagent.models import get_model
+            from minisweagent.environments.docker import DockerEnvironment
+
+            mswe_model = get_model(model_name_full)
+
+            # Log API key source for debugging
+            import os
+            api_key = os.environ.get("ANTHROPIC_API_KEY", "")
+            if api_key:
+                logger.info("ANTHROPIC_API_KEY from env: %s...%s", api_key[:8], api_key[-4:])
+            else:
+                logger.warning("ANTHROPIC_API_KEY not set in environment")
+            # Check mini-swe-agent's config file
+            mswe_env_path = Path.home() / "Library" / "Application Support" / "mini-swe-agent" / ".env"
+            if mswe_env_path.exists():
+                for line in mswe_env_path.read_text().splitlines():
+                    if "ANTHROPIC_API_KEY" in line and not line.strip().startswith("#"):
+                        val = line.split("=", 1)[-1].strip().strip("'\"")
+                        logger.info("mini-swe-agent .env has ANTHROPIC_API_KEY: %s...%s", val[:8], val[-4:])
+                        if val != api_key:
+                            logger.warning("mini-swe-agent .env key DIFFERS from shell env — this key will be used")
+
+            # Build run_args to mount gortex tools into mini-swe-agent's container
+            run_args = ["--rm"]
+            gortex_mode = agent_cfg.get("gortex_mode", "baseline")
+            if gortex_mode in ("native", "native_augment"):
+                gortex_binary = env_cfg.get("gortex_binary")
+                if gortex_binary and Path(gortex_binary).is_file():
+                    abs_binary = str(Path(gortex_binary).resolve())
+                    run_args.append(f"-v={abs_binary}:/usr/local/bin/gortex:ro")
+
+                bridge_dir = Path(__file__).resolve().parent / "bridge"
+                if bridge_dir.is_dir():
+                    for script in bridge_dir.iterdir():
+                        if script.is_file() and not script.name.startswith("__") and not script.name.endswith(".py"):
+                            run_args.append(f"-v={script.resolve()}:/usr/local/bin/{script.name}:ro")
+
+            mswe_env = DockerEnvironment(
+                image=instance_image,
+                executable=env_cfg.get("container_executable", "docker"),
+                cwd="/testbed",
+                run_args=run_args,
+            )
+
+            # For native/native_augment: start eval-server inside mini-swe-agent's container
+            # after it launches, before the agent starts running
+            _start_eval_server_hook = None
+            if gortex_mode in ("native", "native_augment") and gortex_binary and Path(gortex_binary).is_file():
+                eval_port = int(env_cfg.get("eval_server_port", 4747))
+                _start_eval_server_hook = (
+                    f"nohup /usr/local/bin/gortex eval-server "
+                    f"--port {eval_port} --index /testbed "
+                    f"> /tmp/gortex-eval-server.log 2>&1 & "
+                    f"for i in $(seq 1 60); do "
+                    f"  curl -sf http://127.0.0.1:{eval_port}/health >/dev/null 2>&1 && break; "
+                    f"  sleep 2; "
+                    f"done"
+                )
+
+            # Load prompt templates for mini-swe-agent
+            gortex_agent_tmp = GortexAgent.__new__(GortexAgent)
+            gortex_agent_tmp.mode = __import__("agents.gortex_agent", fromlist=["GortexMode"]).GortexMode(
+                agent_cfg.get("gortex_mode", "baseline")
+            )
+            gortex_agent_tmp._system_template = None
+            gortex_agent_tmp._instance_template = None
+            gortex_agent_tmp._load_prompt_templates()
+
+            system_tpl = gortex_agent_tmp.render_system_prompt()
+            instance_tpl = gortex_agent_tmp.render_instance_prompt("{{task}}")
+
+            agent = GortexAgent(
+                config=config,
+                model=mswe_model,
+                env=mswe_env,
+                system_template=system_tpl,
+                instance_template=instance_tpl,
+            )
+        except ImportError:
+            agent = GortexAgent(config=config)
+            _start_eval_server_hook = None
+
+        # --- drive agent ---
+        logger.info("[%s] Starting instance %s", run_id, instance_id)
+
+        # Start eval-server inside mini-swe-agent's container if in native mode
+        if _start_eval_server_hook:
+            try:
+                logger.info("[%s] Starting eval-server in agent container (indexing may take 1-2 min)", run_id)
+                mswe_env.execute(
+                    {"command": _start_eval_server_hook},
+                    timeout=180,  # 3 min for indexing large repos
+                )
+                # Verify it's running
+                health = mswe_env.execute(
+                    {"command": "curl -sf http://127.0.0.1:4747/health 2>/dev/null || echo 'not ready'"},
+                    timeout=10,
+                )
+                logger.info("[%s] Eval-server health: %s", run_id, health.get("output", "")[:200])
+            except Exception as srv_exc:
+                logger.warning("[%s] Eval-server startup failed: %s", run_id, srv_exc)
+
+        task = instance.get("problem_statement", "")
+
+        # Run agent (the agent respects its own step/cost limits)
+        info = agent.run(task)
+
+        # Extract metrics from mini-swe-agent's return value
+        if isinstance(info, dict):
+            result["exit_status"] = info.get("exit_status", "submitted")
+            result["submission"] = info.get("submission", "")
+        else:
+            result["exit_status"] = "submitted"
+
+        # Extract cost/call metrics from mini-swe-agent's agent internals
+        result["cost"] = getattr(agent, "cost", 0.0)
+        result["n_calls"] = getattr(agent, "n_calls", 0)
+        result["n_steps"] = getattr(agent, "n_steps", getattr(agent, "_step_count", 0))
+
+        # Try to get token counts from model stats
+        model_obj = getattr(agent, "model", None)
+        if model_obj:
+            result["tokens_input"] = getattr(model_obj, "tokens_input", 0)
+            result["tokens_output"] = getattr(model_obj, "tokens_output", 0)
+
+        # Overlay with our GortexAgent metrics
+        result["gortex_metrics"] = agent.get_metrics()
+
+        # Also try to get patch from mini-swe-agent's serialized state or env
+        if not result["submission"]:
+            try:
+                serialized = agent.serialize()
+                result["submission"] = serialized.get("info", {}).get("submission", "")
+            except Exception:
+                pass
+        if not result["submission"]:
+            patch = env.extract_patch()
+            result["submission"] = patch or ""
+
+    except KeyboardInterrupt:
+        raise
+    except Exception as exc:
+        logger.error("[%s] Instance %s failed: %s", run_id, instance_id, exc)
+        result["exit_status"] = "error"
+        result["error"] = str(exc)
+
+    finally:
+        result["duration_seconds"] = round(time.time() - start_time, 2)
+
+        # --- save per-instance result JSON ---
+        try:
+            result_file = instance_dir / f"{instance_id}.json"
+            result_file.write_text(json.dumps(result, indent=2, default=str))
+            logger.info("[%s] Result saved: %s", run_id, result_file)
+        except Exception as save_exc:
+            logger.warning("[%s] Failed to save result for %s: %s", run_id, instance_id, save_exc)
+
+        # --- save mini-swe-agent trajectory if available ---
+        if agent is not None:
+            try:
+                trajectory = getattr(agent, "serialize", lambda: None)()
+                if trajectory:
+                    traj_file = instance_dir / f"{instance_id}_trajectory.json"
+                    traj_file.write_text(json.dumps(trajectory, indent=2, default=str))
+                    logger.info("[%s] Trajectory saved: %s", run_id, traj_file)
+            except Exception as traj_exc:
+                logger.debug("[%s] No trajectory for %s: %s", run_id, instance_id, traj_exc)
+
+        # --- log summary line for quick scanning ---
+        patch_len = len(result.get("submission", ""))
+        logger.info(
+            "[%s] %s | status=%s | cost=$%.4f | steps=%s | calls=%s | patch=%d chars | %.1fs",
+            run_id, instance_id,
+            result.get("exit_status", "unknown"),
+            result.get("cost", 0),
+            result.get("n_steps", 0),
+            result.get("n_calls", 0),
+            patch_len,
+            result.get("duration_seconds", 0),
+        )
+
+        # --- teardown ---
+        if env is not None:
+            try:
+                env.teardown()
+            except Exception as teardown_exc:
+                logger.warning(
+                    "[%s] Teardown failed for %s: %s",
+                    run_id, instance_id, teardown_exc,
+                )
+
+    return result
+
+
+def run_configuration(
+    model_name: str,
+    mode_name: str,
+    instances: List[Dict[str, Any]],
+    output_dir: Path,
+    workers: int = 1,
+) -> List[Dict[str, Any]]:
+    """Run a single (model, mode) configuration across all instances.
+
+    Supports sequential (workers=1) or parallel execution via
+    ``concurrent.futures.ThreadPoolExecutor``.
+    """
+    config = _build_config(model_name, mode_name)
+    run_id = generate_run_id(model_name, mode_name)
+    run_dir = output_dir / run_id
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    # Set up file logging for this run
+    file_handler = None
+    try:
+        log_file = run_dir / "run.log"
+        file_handler = logging.FileHandler(str(log_file))
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(logging.Formatter(
+            "%(asctime)s %(name)s %(levelname)s %(message)s"
+        ))
+        logging.getLogger().addHandler(file_handler)
+        logger.info("[%s] Log file: %s", run_id, log_file)
+    except Exception:
+        file_handler = None
+    logger.info("[%s] Config: %s", run_id, json.dumps(config, indent=2, default=str))
+
+    logger.info(
+        "[%s] Running %d instances with %d worker(s)",
+        run_id, len(instances), workers,
+    )
+
+    results: List[Dict[str, Any]] = []
+
+    if workers <= 1:
+        # Sequential execution
+        for instance in instances:
+            result = process_instance(
+                instance, config, output_dir, run_id, model_name, mode_name,
+            )
+            results.append(result)
+    else:
+        # Parallel execution
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = {
+                executor.submit(
+                    process_instance,
+                    instance, config, output_dir, run_id, model_name, mode_name,
+                ): instance["instance_id"]
+                for instance in instances
+            }
+            for future in concurrent.futures.as_completed(futures):
+                iid = futures[future]
+                try:
+                    results.append(future.result())
+                except Exception as exc:
+                    logger.error("[%s] Uncaught error for %s: %s", run_id, iid, exc)
+                    results.append({
+                        "instance_id": iid,
+                        "model": model_name,
+                        "mode": mode_name,
+                        "exit_status": "error",
+                        "error": str(exc),
+                        "submission": "",
+                    })
+
+    # Write run summary with per-instance breakdown
+    summary = {
+        "run_id": run_id,
+        "model": model_name,
+        "mode": mode_name,
+        "total_instances": len(results),
+        "completed": sum(
+            1 for r in results if r.get("exit_status") not in (None, "error", "setup_failure")
+        ),
+        "errors": sum(1 for r in results if r.get("exit_status") == "error"),
+        "setup_failures": sum(1 for r in results if r.get("exit_status") == "setup_failure"),
+        "patches_produced": sum(1 for r in results if r.get("submission")),
+        "total_cost": sum(r.get("cost", 0) for r in results),
+        "total_duration": sum(r.get("duration_seconds", 0) for r in results),
+        "per_instance": [
+            {
+                "instance_id": r.get("instance_id"),
+                "exit_status": r.get("exit_status"),
+                "cost": r.get("cost", 0),
+                "n_steps": r.get("n_steps", 0),
+                "n_calls": r.get("n_calls", 0),
+                "duration_seconds": r.get("duration_seconds", 0),
+                "has_patch": bool(r.get("submission")),
+                "error": r.get("error", ""),
+            }
+            for r in results
+        ],
+    }
+    try:
+        (run_dir / "summary.json").write_text(
+            json.dumps(summary, indent=2, default=str)
+        )
+        logger.info("[%s] Summary saved: %s", run_id, run_dir / "summary.json")
+    except Exception as exc:
+        logger.warning("Failed to write summary: %s", exc)
+
+    # Remove file handler to avoid accumulation across runs
+    if file_handler is not None:
+        logging.getLogger().removeHandler(file_handler)
+        file_handler.close()
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# CLI subcommands
+# ---------------------------------------------------------------------------
+
+def cmd_single(args: argparse.Namespace) -> None:
+    """Handle the ``single`` subcommand."""
+    instances = load_instances(
+        subset=args.subset,
+        split=args.split,
+        slice_spec=args.slice or "",
+        filter_spec=args.filter or "",
+    )
+
+    print(f"\nRunning evaluation: {args.model} + {args.mode}")
+    print(f"  Instances: {len(instances)}")
+    print(f"  Output: {args.output}\n")
+
+    results = run_configuration(
+        args.model, args.mode, instances, Path(args.output), args.workers,
+    )
+
+    _print_summary(results, args.model, args.mode)
+
+
+def cmd_matrix(args: argparse.Namespace) -> None:
+    """Handle the ``matrix`` subcommand."""
+    instances = load_instances(
+        subset=args.subset,
+        split=args.split,
+        slice_spec=args.slice or "",
+        filter_spec=args.filter or "",
+    )
+
+    combos = build_matrix_configs(args.models, args.modes)
+    print(f"\nMatrix evaluation: {len(args.models)} models x {len(args.modes)} modes = {len(combos)} configs")
+    print(f"  Models: {', '.join(args.models)}")
+    print(f"  Modes: {', '.join(args.modes)}")
+    print(f"  Instances per config: {len(instances)}")
+    print(f"  Output: {args.output}\n")
+
+    all_results: Dict[str, List[Dict[str, Any]]] = {}
+    for model_name, mode_name in combos:
+        run_key = f"{model_name}_{mode_name}"
+        print(f"\n━━━ {run_key} ━━━")
+        results = run_configuration(
+            model_name, mode_name, instances, Path(args.output), args.workers,
+        )
+        all_results[run_key] = results
+
+    _print_matrix_summary(all_results)
+
+
+def cmd_debug(args: argparse.Namespace) -> None:
+    """Handle the ``debug`` subcommand — single instance with verbose logging."""
+    # Enable verbose logging
+    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s")
+
+    instances = load_instances(
+        subset=args.subset,
+        split=args.split,
+    )
+
+    # Find the target instance
+    target = None
+    for inst in instances:
+        if inst["instance_id"] == args.instance_id:
+            target = inst
+            break
+
+    if target is None:
+        print(f"Error: instance '{args.instance_id}' not found in {args.subset}/{args.split}")
+        sys.exit(1)
+
+    print(f"\nDebug run: {args.model} + {args.mode}")
+    print(f"  Instance: {args.instance_id}")
+    print(f"  Output: {args.output}\n")
+
+    config = _build_config(args.model, args.mode)
+    run_id = generate_run_id(args.model, args.mode)
+
+    result = process_instance(
+        target, config, Path(args.output), run_id, args.model, args.mode,
+    )
+
+    print(f"\nResult: {json.dumps(result, indent=2, default=str)}")
+
+
+def cmd_list_configs(args: argparse.Namespace) -> None:
+    """Handle the ``list-configs`` subcommand."""
+    from config import list_configs
+
+    configs = list_configs()
+
+    print("\nAvailable configurations:")
+    print(f"\n  Models ({len(configs.get('models', []))}):")
+    for name in configs.get("models", []):
+        print(f"    - {name}")
+
+    print(f"\n  Modes ({len(configs.get('modes', []))}):")
+    for name in configs.get("modes", []):
+        print(f"    - {name}")
+    print()
+
+
+# ---------------------------------------------------------------------------
+# Output helpers
+# ---------------------------------------------------------------------------
+
+def _print_summary(
+    results: List[Dict[str, Any]], model: str, mode: str
+) -> None:
+    """Print a brief summary of a single run."""
+    total = len(results)
+    if total == 0:
+        print("No results.")
+        return
+
+    patches = sum(1 for r in results if r.get("submission"))
+    cost = sum(r.get("cost", 0) for r in results)
+    errors = sum(1 for r in results if r.get("exit_status") == "error")
+
+    print(f"\n{'─' * 50}")
+    print(f"  {model} + {mode}")
+    print(f"  Instances: {total}")
+    print(f"  Patches:   {patches}/{total} ({patches / total * 100:.0f}%)")
+    print(f"  Errors:    {errors}")
+    print(f"  Cost:      ${cost:.2f}")
+    print(f"{'─' * 50}")
+
+
+def _print_matrix_summary(all_results: Dict[str, List[Dict[str, Any]]]) -> None:
+    """Print a comparative summary across all matrix runs."""
+    print(f"\n{'═' * 60}")
+    print("  Matrix Summary")
+    print(f"{'═' * 60}")
+
+    for run_key, results in all_results.items():
+        total = len(results)
+        if total == 0:
+            print(f"  {run_key}: no results")
+            continue
+        patches = sum(1 for r in results if r.get("submission"))
+        cost = sum(r.get("cost", 0) for r in results)
+        print(f"  {run_key}: {patches}/{total} patches, ${cost:.2f} cost")
+
+    print(f"{'═' * 60}")
+
+
+# ---------------------------------------------------------------------------
+# CLI parser
+# ---------------------------------------------------------------------------
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the argparse CLI parser with subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="gortex-eval",
+        description="SWE-bench evaluation harness for Gortex code intelligence",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="Enable verbose logging"
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # --- single ---
+    p_single = subparsers.add_parser("single", help="Run a single (model, mode) configuration")
+    p_single.add_argument("-m", "--model", required=True, help="Model config name")
+    p_single.add_argument("--mode", default="baseline", help="Evaluation mode (default: baseline)")
+    p_single.add_argument("--subset", default=DEFAULT_SUBSET, help="SWE-bench subset: lite, verified, full")
+    p_single.add_argument("--split", default=DEFAULT_SPLIT, help="Dataset split")
+    p_single.add_argument("--slice", default="", help="Slice spec (e.g., '0:5', ':3')")
+    p_single.add_argument("--filter", default="", help="Filter instance IDs by regex")
+    p_single.add_argument("-w", "--workers", type=int, default=1, help="Parallel workers (default: 1)")
+    p_single.add_argument("-o", "--output", default=str(DEFAULT_OUTPUT_DIR), help="Output directory")
+    p_single.set_defaults(func=cmd_single)
+
+    # --- matrix ---
+    p_matrix = subparsers.add_parser("matrix", help="Run full model × mode evaluation matrix")
+    p_matrix.add_argument("--models", nargs="+", required=True, help="Model config names")
+    p_matrix.add_argument("--modes", nargs="+", required=True, help="Mode config names")
+    p_matrix.add_argument("--subset", default=DEFAULT_SUBSET, help="SWE-bench subset")
+    p_matrix.add_argument("--split", default=DEFAULT_SPLIT, help="Dataset split")
+    p_matrix.add_argument("--slice", default="", help="Slice spec")
+    p_matrix.add_argument("--filter", default="", help="Filter instance IDs by regex")
+    p_matrix.add_argument("-w", "--workers", type=int, default=1, help="Parallel workers per config")
+    p_matrix.add_argument("-o", "--output", default=str(DEFAULT_OUTPUT_DIR), help="Output directory")
+    p_matrix.set_defaults(func=cmd_matrix)
+
+    # --- debug ---
+    p_debug = subparsers.add_parser("debug", help="Debug a single instance with verbose logging")
+    p_debug.add_argument("-m", "--model", required=True, help="Model config name")
+    p_debug.add_argument("--mode", default="baseline", help="Evaluation mode")
+    p_debug.add_argument("-i", "--instance-id", required=True, help="SWE-bench instance ID")
+    p_debug.add_argument("--subset", default=DEFAULT_SUBSET, help="SWE-bench subset")
+    p_debug.add_argument("--split", default=DEFAULT_SPLIT, help="Dataset split")
+    p_debug.add_argument("-o", "--output", default=str(DEFAULT_OUTPUT_DIR), help="Output directory")
+    p_debug.set_defaults(func=cmd_debug)
+
+    # --- list-configs ---
+    p_list = subparsers.add_parser("list-configs", help="Show available model/mode configs")
+    p_list.set_defaults(func=cmd_list_configs)
+
+    return parser
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s")
+    else:
+        logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(0)
+
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

eval/tests/__init__.py

@@ -0,0 +1 @@
+"""Eval framework test suite."""

eval/tests/test_augmentation_properties.py

@@ -0,0 +1,191 @@
+"""Property-based tests for eval.augmentation module.
+
+Feature: eval-framework
+Uses hypothesis to verify augmentation triggering and timeout properties.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict
+from unittest.mock import MagicMock, patch
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from augmentation import augment_grep_output
+
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Printable text that could appear in grep output lines.
+_grep_line_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z"), whitelist_characters=(":", "/", ".", "_", "-")),
+    min_size=1,
+    max_size=80,
+)
+
+# Multi-line grep output.
+_grep_output_st = st.lists(_grep_line_st, min_size=1, max_size=10).map("\n".join)
+
+# Search patterns: printable, no quotes, no leading special chars.
+_pattern_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_",)),
+    min_size=1,
+    max_size=30,
+).filter(lambda s: not s.startswith(("/", ".", "-")))
+
+# Minimum pattern length config values.
+_min_len_st = st.integers(min_value=1, max_value=20)
+
+
+def _make_grep_command(pattern: str) -> str:
+    """Build a grep command string with the given pattern."""
+    return f'grep -rn "{pattern}" .'
+
+
+# ---------------------------------------------------------------------------
+# Property 13: Augmentation triggering rules
+# Feature: eval-framework, Property 13: Augmentation triggering rules
+# **Validates: Requirements 9.1, 9.4**
+# ---------------------------------------------------------------------------
+
+
+class TestAugmentationTriggeringRules:
+    """For any grep/rg command in native_augment mode, if the extracted search
+    pattern has length >= the configured minimum, augmentation SHALL be
+    attempted. If the pattern length is below the minimum, augmentation SHALL
+    be skipped and the original output returned."""
+
+    @given(pattern=_pattern_st, min_len=_min_len_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_short_pattern_skips_augmentation(
+        self, pattern: str, min_len: int, raw_output: str
+    ) -> None:
+        """Pattern shorter than minimum → augmentation skipped, original output returned."""
+        assume(len(pattern) < min_len)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": min_len}
+
+        # urlopen should never be called when pattern is too short.
+        with patch("augmentation.urllib.request.urlopen") as mock_urlopen:
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        mock_urlopen.assert_not_called()
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, min_len=_min_len_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_long_pattern_attempts_augmentation(
+        self, pattern: str, min_len: int, raw_output: str
+    ) -> None:
+        """Pattern >= minimum length → augmentation attempted (HTTP call made)."""
+        assume(len(pattern) >= min_len)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": min_len}
+
+        # Mock urlopen to return a valid augmentation response.
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({
+            "callers": [{"name": "Caller", "location": "file.go:1"}],
+        }).encode("utf-8")
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+
+        with patch("augmentation.urllib.request.urlopen", return_value=mock_response) as mock_urlopen:
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        mock_urlopen.assert_called_once()
+        # Result should contain the Gortex annotation.
+        assert "[Gortex]" in result
+
+
+# ---------------------------------------------------------------------------
+# Property 14: Augmentation timeout preserves output
+# Feature: eval-framework, Property 14: Augmentation timeout preserves output
+# **Validates: Requirements 9.3**
+# ---------------------------------------------------------------------------
+
+
+class TestAugmentationTimeoutPreservesOutput:
+    """For any grep output where augmentation times out or returns nothing,
+    returned output is identical to original."""
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_timeout_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When the augmentation endpoint times out, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            side_effect=TimeoutError("timed out"),
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_connection_error_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When the augmentation endpoint is unreachable, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        import urllib.error
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            side_effect=urllib.error.URLError("connection refused"),
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_empty_response_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When augmentation returns no useful context, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        # Return empty annotations (no callers/callees/flows).
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({}).encode("utf-8")
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            return_value=mock_response,
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output

eval/tests/test_bridge_properties.py

@@ -0,0 +1,210 @@
+"""Property-based tests for tool bridge output format.
+
+Feature: eval-framework, Property 15: Tool bridge output format
+**Validates: Requirements 8.4, 8.5**
+
+For any valid Gortex tool JSON response, formatted output is plain text
+(not raw JSON) and contains next-step hints guiding the agent toward
+effective tool chaining.
+
+Since bridge scripts require a running eval-server for full execution,
+we test the formatting properties by:
+1. Verifying scripts contain "Next steps" hints in their output
+2. Piping mock JSON through the jq formatting logic extracted from the scripts
+3. Using hypothesis to generate various JSON response shapes
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import shutil
+from pathlib import Path
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+BRIDGE_DIR = Path(__file__).resolve().parent.parent / "bridge"
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+)
+
+# User-facing scripts (gortex-augment is an internal helper without next-step hints)
+USER_FACING_SCRIPTS = [s for s in BRIDGE_SCRIPTS if s.name != "gortex-augment"]
+
+# The jq filter used by bridge scripts to format MCP responses.
+# Must check `type` before accessing keys to avoid "Cannot index array" errors.
+JQ_FORMAT_FILTER = r'''
+if type == "array" then
+    .[] | if .id then "\(.id)  \(.kind // "")  \(.file // "")" else tostring end
+elif type == "object" and .content then
+    .content[] | select(.type == "text") | .text
+elif type == "object" and .error then
+    "Error: \(.error)"
+else
+    tostring
+end
+'''
+
+HAS_JQ = shutil.which("jq") is not None
+
+
+# ---------------------------------------------------------------------------
+# Strategies for generating Gortex-like JSON responses
+# ---------------------------------------------------------------------------
+
+_safe_text = st.text(
+    alphabet=st.characters(
+        whitelist_categories=("L", "N", "P", "Z"),
+        blacklist_characters=("\x00", "{", "["),
+    ),
+    min_size=1,
+    max_size=100,
+).filter(lambda s: s.strip())
+
+# MCP-style content response: {"content": [{"type": "text", "text": "..."}]}
+_mcp_content_st = st.builds(
+    lambda texts: {"content": [{"type": "text", "text": t} for t in texts]},
+    texts=st.lists(_safe_text, min_size=1, max_size=3),
+)
+
+# Array of symbol-like objects
+_symbol_obj_st = st.fixed_dictionaries({
+    "id": _safe_text,
+    "kind": st.sampled_from(["function", "method", "type", "interface", "variable"]),
+    "file": _safe_text,
+})
+
+_symbol_array_st = st.lists(_symbol_obj_st, min_size=1, max_size=5)
+
+# Error response
+_error_response_st = st.builds(
+    lambda msg: {"error": msg},
+    msg=_safe_text,
+)
+
+# Combined strategy for any valid Gortex response shape
+_gortex_response_st = st.one_of(
+    _mcp_content_st,
+    _symbol_array_st,
+    _error_response_st,
+)
+
+
+def _run_jq(json_input: str, jq_filter: str) -> subprocess.CompletedProcess:
+    """Run jq with the given filter on the input JSON string."""
+    return subprocess.run(
+        ["jq", "-r", jq_filter],
+        input=json_input,
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Property 15: Tool bridge output format
+# ---------------------------------------------------------------------------
+
+
+class TestToolBridgeOutputFormat:
+    """For any valid Gortex tool JSON response, formatted output is plain text
+    (not raw JSON) and contains next-step hints."""
+
+    def test_all_scripts_contain_next_steps_section(self) -> None:
+        """Every user-facing bridge script must include a 'Next steps' section."""
+        for script in USER_FACING_SCRIPTS:
+            content = script.read_text()
+            assert "Next steps" in content, (
+                f"{script.name} does not contain 'Next steps' hints"
+            )
+
+    def test_all_scripts_contain_gortex_tool_hints(self) -> None:
+        """Next-step hints should reference other gortex-* tools for chaining."""
+        for script in USER_FACING_SCRIPTS:
+            content = script.read_text()
+            # Each script should suggest at least one other gortex-* tool
+            hint_tools = [
+                "gortex-search", "gortex-context", "gortex-impact",
+                "gortex-overview", "gortex-usages", "gortex-augment",
+            ]
+            other_tools = [t for t in hint_tools if t != script.name]
+            has_hint = any(tool in content for tool in other_tools)
+            assert has_hint, (
+                f"{script.name} does not reference any other gortex-* tools in hints"
+            )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_mcp_content_st)
+    @settings(max_examples=100, deadline=None)
+    def test_mcp_content_formatted_as_plain_text(self, response: dict) -> None:
+        """MCP content responses are formatted as plain text, not raw JSON."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, "Formatted output should not be empty"
+
+        # Output should NOT look like raw JSON (no leading { or [)
+        assert not output.startswith("{"), (
+            f"Output looks like raw JSON object: {output[:80]}"
+        )
+        assert not output.startswith("["), (
+            f"Output looks like raw JSON array: {output[:80]}"
+        )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(symbols=_symbol_array_st)
+    @settings(max_examples=100, deadline=None)
+    def test_symbol_array_formatted_as_plain_text(self, symbols: list) -> None:
+        """Symbol array responses are formatted as readable lines, not JSON."""
+        json_str = json.dumps(symbols)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, "Formatted output should not be empty"
+
+        # Each symbol should produce a line with its id
+        lines = output.split("\n")
+        assert len(lines) >= 1, "Expected at least one output line per symbol"
+
+        # Output should not be raw JSON
+        for line in lines:
+            stripped = line.strip()
+            if stripped:
+                assert not stripped.startswith("{"), (
+                    f"Line looks like raw JSON: {stripped[:80]}"
+                )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_error_response_st)
+    @settings(max_examples=100, deadline=None)
+    def test_error_response_formatted_as_plain_text(self, response: dict) -> None:
+        """Error responses are formatted as 'Error: ...' text, not raw JSON."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert output.startswith("Error:"), (
+            f"Error response should start with 'Error:' but got: {output[:80]}"
+        )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_gortex_response_st)
+    @settings(max_examples=100, deadline=None)
+    def test_any_response_produces_non_empty_output(self, response) -> None:
+        """Any valid Gortex response shape produces non-empty formatted output."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, (
+            f"Expected non-empty output for response: {json_str[:120]}"
+        )

eval/tests/test_bridge_syntax.py

@@ -0,0 +1,48 @@
+"""Bash syntax validation tests for tool bridge scripts.
+
+Feature: eval-framework
+Verifies all bridge scripts in eval/bridge/ pass `bash -n` (syntax check).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+# All bridge scripts in eval/bridge/ (no .py files, no __pycache__)
+BRIDGE_DIR = Path(__file__).resolve().parent.parent / "bridge"
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+)
+
+
+@pytest.fixture(params=BRIDGE_SCRIPTS, ids=lambda p: p.name)
+def bridge_script(request: pytest.FixtureRequest) -> Path:
+    return request.param
+
+
+def test_bridge_scripts_discovered() -> None:
+    """Sanity check: we found at least the 6 expected bridge scripts."""
+    assert len(BRIDGE_SCRIPTS) >= 6, (
+        f"Expected at least 6 bridge scripts, found {len(BRIDGE_SCRIPTS)}: "
+        f"{[p.name for p in BRIDGE_SCRIPTS]}"
+    )
+
+
+def test_bash_syntax_valid(bridge_script: Path) -> None:
+    """Each bridge script must pass bash -n (syntax check)."""
+    result = subprocess.run(
+        ["bash", "-n", str(bridge_script)],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"bash -n failed for {bridge_script.name}:\n"
+        f"stderr: {result.stderr}\n"
+        f"stdout: {result.stdout}"
+    )

eval/tests/test_config.py

@@ -0,0 +1,200 @@
+"""Unit tests for eval.config module."""
+
+from __future__ import annotations
+
+import textwrap
+from pathlib import Path
+
+import pytest
+import yaml
+
+from config import (
+    list_configs,
+    load_mode_config,
+    load_model_config,
+    merge_configs,
+    validate_config,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def tmp_configs(tmp_path, monkeypatch):
+    """Create a temporary configs directory and patch _CONFIGS_DIR."""
+    models_dir = tmp_path / "models"
+    modes_dir = tmp_path / "modes"
+    models_dir.mkdir()
+    modes_dir.mkdir()
+
+    import config as config_mod
+
+    monkeypatch.setattr(config_mod, "_CONFIGS_DIR", tmp_path)
+    return tmp_path
+
+
+def _write_yaml(path: Path, data: dict) -> None:
+    with open(path, "w") as f:
+        yaml.safe_dump(data, f)
+
+
+# ---------------------------------------------------------------------------
+# load_model_config
+# ---------------------------------------------------------------------------
+
+
+class TestLoadModelConfig:
+    def test_loads_valid_yaml(self, tmp_configs):
+        data = {"model": {"model_name": "test-model", "cost_tracking": "ignore_errors"}}
+        _write_yaml(tmp_configs / "models" / "test.yaml", data)
+        result = load_model_config("test")
+        assert result == data
+
+    def test_missing_config_raises(self, tmp_configs):
+        with pytest.raises(FileNotFoundError, match="Model config not found"):
+            load_model_config("nonexistent")
+
+    def test_empty_yaml_returns_empty_dict(self, tmp_configs):
+        (tmp_configs / "models" / "empty.yaml").write_text("")
+        result = load_model_config("empty")
+        assert result == {}
+
+
+# ---------------------------------------------------------------------------
+# load_mode_config
+# ---------------------------------------------------------------------------
+
+
+class TestLoadModeConfig:
+    def test_loads_valid_yaml(self, tmp_configs):
+        data = {
+            "agent": {"agent_class": "eval.agents.gortex_agent.GortexAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        _write_yaml(tmp_configs / "modes" / "baseline.yaml", data)
+        result = load_mode_config("baseline")
+        assert result == data
+
+    def test_missing_config_raises(self, tmp_configs):
+        with pytest.raises(FileNotFoundError, match="Mode config not found"):
+            load_mode_config("nonexistent")
+
+
+# ---------------------------------------------------------------------------
+# merge_configs
+# ---------------------------------------------------------------------------
+
+
+class TestMergeConfigs:
+    def test_mode_overrides_model_on_conflict(self):
+        model = {"model": {"model_name": "old"}, "shared": "model_val"}
+        mode = {"shared": "mode_val"}
+        result = merge_configs(model, mode)
+        assert result["shared"] == "mode_val"
+        assert result["model"]["model_name"] == "old"
+
+    def test_deep_merge_nested_dicts(self):
+        model = {"agent": {"step_limit": 30, "cost_limit": 3.0}}
+        mode = {"agent": {"step_limit": 50, "extra": True}}
+        result = merge_configs(model, mode)
+        assert result["agent"]["step_limit"] == 50
+        assert result["agent"]["cost_limit"] == 3.0
+        assert result["agent"]["extra"] is True
+
+    def test_unique_keys_preserved(self):
+        model = {"model": {"model_name": "test"}}
+        mode = {"agent": {"agent_class": "MyAgent"}}
+        result = merge_configs(model, mode)
+        assert result["model"]["model_name"] == "test"
+        assert result["agent"]["agent_class"] == "MyAgent"
+
+    def test_empty_configs(self):
+        assert merge_configs({}, {}) == {}
+        assert merge_configs({"a": 1}, {}) == {"a": 1}
+        assert merge_configs({}, {"b": 2}) == {"b": 2}
+
+    def test_scalar_override_replaces_dict(self):
+        model = {"key": {"nested": "value"}}
+        mode = {"key": "scalar"}
+        result = merge_configs(model, mode)
+        assert result["key"] == "scalar"
+
+
+# ---------------------------------------------------------------------------
+# validate_config
+# ---------------------------------------------------------------------------
+
+
+class TestValidateConfig:
+    def test_valid_config_passes(self):
+        config = {
+            "model": {"model_name": "test"},
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        validate_config(config)  # should not raise
+
+    def test_missing_single_field(self):
+        config = {
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        with pytest.raises(ValueError, match="model.model_name"):
+            validate_config(config)
+
+    def test_missing_multiple_fields(self):
+        with pytest.raises(ValueError) as exc_info:
+            validate_config({})
+        msg = str(exc_info.value)
+        assert "model.model_name" in msg
+        assert "agent.agent_class" in msg
+        assert "environment.environment_class" in msg
+
+    def test_missing_nested_key(self):
+        config = {
+            "model": {},  # model_name missing inside model dict
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        with pytest.raises(ValueError, match="model.model_name"):
+            validate_config(config)
+
+
+# ---------------------------------------------------------------------------
+# list_configs
+# ---------------------------------------------------------------------------
+
+
+class TestListConfigs:
+    def test_discovers_yaml_files(self, tmp_configs):
+        _write_yaml(tmp_configs / "models" / "claude-sonnet.yaml", {"model": {}})
+        _write_yaml(tmp_configs / "models" / "claude-haiku.yaml", {"model": {}})
+        _write_yaml(tmp_configs / "modes" / "baseline.yaml", {"agent": {}})
+        _write_yaml(tmp_configs / "modes" / "native.yaml", {"agent": {}})
+
+        result = list_configs()
+        assert "claude-haiku" in result["models"]
+        assert "claude-sonnet" in result["models"]
+        assert "baseline" in result["modes"]
+        assert "native" in result["modes"]
+
+    def test_excludes_non_yaml_files(self, tmp_configs):
+        _write_yaml(tmp_configs / "models" / "valid.yaml", {"model": {}})
+        (tmp_configs / "models" / ".gitkeep").write_text("")
+        (tmp_configs / "models" / "readme.txt").write_text("not yaml")
+
+        result = list_configs()
+        assert result["models"] == ["valid"]
+
+    def test_empty_directories(self, tmp_configs):
+        result = list_configs()
+        assert result == {"models": [], "modes": []}
+
+    def test_returns_sorted_names(self, tmp_configs):
+        for name in ["zebra", "alpha", "middle"]:
+            _write_yaml(tmp_configs / "models" / f"{name}.yaml", {})
+        result = list_configs()
+        assert result["models"] == ["alpha", "middle", "zebra"]

eval/tests/test_config_properties.py

@@ -0,0 +1,210 @@
+"""Property-based tests for eval.config module.
+
+Feature: eval-framework
+Uses hypothesis to verify config merge and validation properties.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from config import merge_configs, validate_config
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Keys: non-empty strings without dots (dots are used as path separators in
+# validate_config, so keeping keys simple avoids confusion).
+_key_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+    min_size=1,
+    max_size=12,
+)
+
+# Leaf values: scalars that YAML configs typically hold.
+_leaf_st = st.one_of(
+    st.text(min_size=0, max_size=30),
+    st.integers(min_value=-1000, max_value=1000),
+    st.floats(allow_nan=False, allow_infinity=False, min_value=-1e6, max_value=1e6),
+    st.booleans(),
+    st.none(),
+)
+
+# Shallow config dict (1 level deep) — sufficient for merge precedence tests.
+_flat_dict_st = st.dictionaries(keys=_key_st, values=_leaf_st, max_size=8)
+
+# Nested config dict (up to 2 levels) — mirrors real YAML configs.
+_nested_value_st = st.one_of(
+    _leaf_st,
+    st.dictionaries(keys=_key_st, values=_leaf_st, max_size=5),
+)
+_config_dict_st = st.dictionaries(keys=_key_st, values=_nested_value_st, max_size=8)
+
+
+# Required field paths used by validate_config.
+_REQUIRED_FIELDS = [
+    "model.model_name",
+    "agent.agent_class",
+    "environment.environment_class",
+]
+
+
+def _build_full_config() -> dict[str, Any]:
+    """Return a config dict that passes validation (all required fields present)."""
+    return {
+        "model": {"model_name": "test-model"},
+        "agent": {"agent_class": "eval.agents.TestAgent"},
+        "environment": {"environment_class": "eval.environments.TestEnv"},
+    }
+
+
+def _set_nested(d: dict, dotted_key: str, value: Any) -> None:
+    """Set a value in a nested dict using dot notation."""
+    parts = dotted_key.split(".")
+    current = d
+    for part in parts[:-1]:
+        current = current.setdefault(part, {})
+    current[parts[-1]] = value
+
+
+# ---------------------------------------------------------------------------
+# Property 10: Config merge precedence
+# Feature: eval-framework, Property 10: Config merge precedence
+# **Validates: Requirements 10.1, 10.2**
+# ---------------------------------------------------------------------------
+
+
+class TestConfigMergePrecedence:
+    """For any two config dicts, mode values override model values on shared
+    keys; unique keys preserved."""
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_mode_overrides_model_on_shared_keys(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """Shared top-level keys take the mode value (or deep-merged sub-dict)."""
+        merged = merge_configs(model, mode)
+
+        for key in mode:
+            if key in model:
+                model_val = model[key]
+                mode_val = mode[key]
+                merged_val = merged[key]
+
+                if isinstance(model_val, dict) and isinstance(mode_val, dict):
+                    # When both are dicts, mode sub-keys override model sub-keys.
+                    for sub_key in mode_val:
+                        assert merged_val[sub_key] == mode_val[sub_key]
+                else:
+                    # Scalar or type mismatch: mode wins entirely.
+                    assert merged_val == mode_val
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_unique_keys_preserved(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """Keys present in only one config appear unchanged in the merged result."""
+        merged = merge_configs(model, mode)
+
+        # Model-only keys preserved.
+        for key in model:
+            if key not in mode:
+                assert merged[key] == model[key]
+
+        # Mode-only keys preserved.
+        for key in mode:
+            if key not in model:
+                assert merged[key] == mode[key]
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_merged_contains_all_keys(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """The merged dict contains every key from both inputs."""
+        merged = merge_configs(model, mode)
+        assert set(merged.keys()) == set(model.keys()) | set(mode.keys())
+
+
+# ---------------------------------------------------------------------------
+# Property 11: Config validation catches missing required fields
+# Feature: eval-framework, Property 11: Config validation catches missing required fields
+# **Validates: Requirements 10.3**
+# ---------------------------------------------------------------------------
+
+# Strategy: pick a non-empty subset of required fields to remove.
+_required_subsets_st = st.lists(
+    st.sampled_from(_REQUIRED_FIELDS),
+    min_size=1,
+    max_size=len(_REQUIRED_FIELDS),
+    unique=True,
+)
+
+
+class TestConfigValidationMissingFields:
+    """For any merged config missing one or more of (model_name, agent_class,
+    environment_class), validation fails naming the missing field(s)."""
+
+    @given(missing_fields=_required_subsets_st)
+    @settings(max_examples=100)
+    def test_validation_fails_naming_missing_fields(
+        self, missing_fields: list[str]
+    ) -> None:
+        """Removing any subset of required fields causes ValueError listing them."""
+        config = _build_full_config()
+
+        # Remove the selected required fields.
+        for field in missing_fields:
+            parts = field.split(".")
+            section = parts[0]
+            key = parts[1]
+            if section in config and isinstance(config[section], dict):
+                config[section].pop(key, None)
+
+        with pytest.raises(ValueError, match="Missing required config fields") as exc_info:
+            validate_config(config)
+
+        error_msg = str(exc_info.value)
+        for field in missing_fields:
+            assert field in error_msg, (
+                f"Expected '{field}' to be named in error but got: {error_msg}"
+            )
+
+    @given(missing_fields=_required_subsets_st)
+    @settings(max_examples=100)
+    def test_validation_only_reports_actually_missing_fields(
+        self, missing_fields: list[str]
+    ) -> None:
+        """Fields that ARE present should NOT appear in the error message."""
+        config = _build_full_config()
+        present_fields = [f for f in _REQUIRED_FIELDS if f not in missing_fields]
+
+        for field in missing_fields:
+            parts = field.split(".")
+            section = parts[0]
+            key = parts[1]
+            if section in config and isinstance(config[section], dict):
+                config[section].pop(key, None)
+
+        with pytest.raises(ValueError) as exc_info:
+            validate_config(config)
+
+        error_msg = str(exc_info.value)
+        for field in present_fields:
+            assert field not in error_msg, (
+                f"Field '{field}' is present but was reported as missing: {error_msg}"
+            )
+
+    @given(st.just(_build_full_config()))
+    @settings(max_examples=10)
+    def test_complete_config_passes_validation(self, config: dict[str, Any]) -> None:
+        """A config with all required fields should pass validation."""
+        validate_config(config)  # should not raise

eval/tests/test_docker_integration.py

@@ -0,0 +1,166 @@
+"""Docker container setup integration test.
+
+Feature: eval-framework
+Tests the GortexDockerEnvironment setup/teardown lifecycle with a real
+Docker daemon. Skipped when Docker is not available.
+
+This test validates:
+- Container launch with a lightweight image
+- Gortex binary copy into container
+- Eval-server health check
+- Container teardown and cleanup
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from environments.gortex_docker import GortexDockerEnvironment, _make_cache_key
+
+
+# --- Docker availability check ---
+
+def _docker_available() -> bool:
+    """Check if Docker daemon is accessible."""
+    try:
+        import docker
+        client = docker.from_env()
+        client.ping()
+        client.close()
+        return True
+    except Exception:
+        return False
+
+
+docker_available = _docker_available()
+
+
+# --- Integration tests (require Docker) ---
+
+@pytest.mark.skipif(
+    not docker_available,
+    reason="Docker daemon not available",
+)
+class TestDockerContainerLifecycle:
+    """Integration tests that exercise real Docker container lifecycle.
+
+    These tests require a running Docker daemon and will pull/use
+    lightweight images. They are skipped in CI environments without Docker.
+    """
+
+    def test_setup_teardown_gortex_disabled(self) -> None:
+        """Launch a container with gortex disabled, verify it runs, teardown."""
+        env = GortexDockerEnvironment(
+            image="alpine:latest",
+            enable_gortex=False,
+            instance_id="integration-test-no-gortex",
+        )
+        try:
+            result = env.setup()
+            # setup should succeed (no failure dict returned)
+            if result is not None:
+                pytest.skip(f"Container setup failed: {result.get('setup_error', 'unknown')}")
+
+            assert env._container is not None
+            assert env.is_ready
+
+            # Run a simple command inside the container
+            code, output = env.exec_run("echo hello-from-container")
+            assert code == 0
+            assert "hello-from-container" in output
+        finally:
+            env.teardown()
+            assert env._container is None
+
+    def test_extract_patch_empty_repo(self) -> None:
+        """Extract patch from a container with no git changes returns empty."""
+        env = GortexDockerEnvironment(
+            image="alpine:latest",
+            enable_gortex=False,
+            instance_id="integration-test-patch",
+        )
+        try:
+            result = env.setup()
+            if result is not None:
+                pytest.skip(f"Container setup failed: {result.get('setup_error', 'unknown')}")
+
+            # No git repo in alpine, so extract_patch should return empty
+            patch = env.extract_patch()
+            assert patch == ""
+        finally:
+            env.teardown()
+
+
+# --- Mock-based tests (always run, no Docker required) ---
+
+class TestDockerEnvironmentMocked:
+    """Tests that verify Docker integration logic using mocks.
+
+    These always run regardless of Docker availability.
+    """
+
+    @patch("environments.gortex_docker.docker")
+    def test_full_lifecycle_gortex_disabled(self, mock_docker) -> None:
+        """Verify setup → exec → extract_patch → teardown with mocked Docker."""
+        mock_client = MagicMock()
+        mock_container = MagicMock()
+        mock_container.short_id = "abc123"
+        mock_container.exec_run.return_value = (0, b"hello\n")
+        mock_client.containers.run.return_value = mock_container
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            enable_gortex=False,
+            instance_id="mock-lifecycle",
+        )
+
+        # Setup
+        result = env.setup()
+        assert result is None
+        assert env._container is mock_container
+
+        # Exec
+        code, output = env.exec_run("echo hello")
+        assert code == 0
+        assert output == "hello\n"
+
+        # Teardown
+        env.teardown()
+        mock_container.stop.assert_called_once()
+        mock_container.remove.assert_called_once()
+        assert env._container is None
+
+    @patch("environments.gortex_docker.docker")
+    def test_setup_failure_records_error(self, mock_docker) -> None:
+        """Verify that container launch failure is properly recorded."""
+        mock_client = MagicMock()
+        mock_client.containers.run.side_effect = RuntimeError("Docker daemon not running")
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            instance_id="fail-test",
+        )
+        result = env.setup()
+        assert result is not None
+        assert result["exit_status"] == "setup_failure"
+        assert "fail-test" in result["instance_id"]
+
+    def test_cache_key_determinism(self) -> None:
+        """Cache key for same inputs is always the same."""
+        k1 = _make_cache_key("repo", "abc123")
+        k2 = _make_cache_key("repo", "abc123")
+        assert k1 == k2
+
+    def test_cache_key_uniqueness(self) -> None:
+        """Different inputs produce different cache keys."""
+        k1 = _make_cache_key("repo_a", "commit1")
+        k2 = _make_cache_key("repo_b", "commit2")
+        assert k1 != k2

eval/tests/test_gortex_docker.py

@@ -0,0 +1,237 @@
+"""Unit tests for eval/environments/gortex_docker.py.
+
+Tests focus on pure logic (cache key, failure recording, properties)
+and mock Docker interactions to avoid requiring a running Docker daemon.
+"""
+
+from __future__ import annotations
+
+import io
+import tarfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from environments.gortex_docker import (
+    DEFAULT_EVAL_SERVER_PORT,
+    DEFAULT_GORTEX_TIMEOUT,
+    GortexDockerEnvironment,
+    _make_cache_key,
+)
+
+
+# -- _make_cache_key ---------------------------------------------------------
+
+class TestMakeCacheKey:
+    def test_basic(self):
+        assert _make_cache_key("django", "abc123") == "django_abc123"
+
+    def test_slash_in_repo_name(self):
+        assert _make_cache_key("django/django", "abc123") == "django__django_abc123"
+
+    def test_deterministic(self):
+        k1 = _make_cache_key("repo", "commit")
+        k2 = _make_cache_key("repo", "commit")
+        assert k1 == k2
+
+    def test_different_inputs_different_keys(self):
+        k1 = _make_cache_key("repo_a", "commit1")
+        k2 = _make_cache_key("repo_b", "commit2")
+        assert k1 != k2
+
+
+# -- GortexDockerEnvironment init -------------------------------------------
+
+class TestInit:
+    def test_defaults(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.image == "test:latest"
+        assert env.enable_gortex is True
+        assert env.gortex_timeout == DEFAULT_GORTEX_TIMEOUT
+        assert env.eval_server_port == DEFAULT_EVAL_SERVER_PORT
+        assert env._container is None
+        assert env._gortex_ready is False
+
+    def test_gortex_disabled(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        assert env.enable_gortex is False
+
+    def test_custom_params(self):
+        env = GortexDockerEnvironment(
+            image="swe:v1",
+            gortex_binary="/tmp/gortex",
+            gortex_timeout=60,
+            eval_server_port=9999,
+            cache_dir="/tmp/cache",
+            instance_id="django__django-1234",
+        )
+        assert env.gortex_binary == Path("/tmp/gortex")
+        assert env.gortex_timeout == 60
+        assert env.eval_server_port == 9999
+        assert env.cache_dir == Path("/tmp/cache")
+        assert env.instance_id == "django__django-1234"
+
+
+# -- setup / failure recording -----------------------------------------------
+
+class TestRecordFailure:
+    def test_returns_failure_dict(self):
+        env = GortexDockerEnvironment(image="test:latest", instance_id="test-123")
+        result = env._record_failure("something broke")
+        assert result["exit_status"] == "setup_failure"
+        assert result["instance_id"] == "test-123"
+        assert "something broke" in result["setup_error"]
+        assert result["submission"] == ""
+
+    def test_sets_setup_error(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        env._record_failure("timeout")
+        assert env.setup_error == "timeout"
+
+
+# -- is_ready property -------------------------------------------------------
+
+class TestIsReady:
+    def test_not_ready_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.is_ready is False
+
+    def test_ready_when_gortex_disabled(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        env._container = MagicMock()  # simulate running container
+        assert env.is_ready is True
+
+    def test_not_ready_gortex_enabled_but_not_setup(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=True)
+        env._container = MagicMock()
+        assert env.is_ready is False
+
+    def test_ready_gortex_enabled_and_setup(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=True)
+        env._container = MagicMock()
+        env._gortex_ready = True
+        assert env.is_ready is True
+
+
+# -- extract_patch -----------------------------------------------------------
+
+class TestExtractPatch:
+    def test_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.extract_patch() == ""
+
+    def test_successful_diff(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"diff --git a/foo.py b/foo.py\n+hello\n")
+        env._container = mock_container
+        patch = env.extract_patch()
+        assert "diff --git" in patch
+        assert "+hello" in patch
+
+    def test_failed_diff(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (1, b"error")
+        env._container = mock_container
+        assert env.extract_patch() == ""
+
+    def test_exception_returns_empty(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.side_effect = RuntimeError("boom")
+        env._container = mock_container
+        assert env.extract_patch() == ""
+
+
+# -- teardown ----------------------------------------------------------------
+
+class TestTeardown:
+    def test_teardown_stops_and_removes(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_client = MagicMock()
+        env._container = mock_container
+        env._client = mock_client
+        env.teardown()
+        mock_container.stop.assert_called_once()
+        mock_container.remove.assert_called_once()
+        mock_client.close.assert_called_once()
+        assert env._container is None
+        assert env._client is None
+
+    def test_teardown_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        env.teardown()  # should not raise
+
+
+# -- exec_run ----------------------------------------------------------------
+
+class TestExecRun:
+    def test_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        code, output = env.exec_run("echo hello")
+        assert code == 1
+        assert "not running" in output.lower()
+
+    def test_string_command_wrapped_in_bash(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"hello\n")
+        env._container = mock_container
+        code, output = env.exec_run("echo hello")
+        assert code == 0
+        assert output == "hello\n"
+        # Verify it was wrapped in bash -c
+        call_args = mock_container.exec_run.call_args
+        assert call_args[0][0] == ["bash", "-c", "echo hello"]
+
+    def test_list_command_passed_directly(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"ok")
+        env._container = mock_container
+        code, output = env.exec_run(["ls", "-la"])
+        assert code == 0
+        call_args = mock_container.exec_run.call_args
+        assert call_args[0][0] == ["ls", "-la"]
+
+
+# -- setup with gortex disabled ---------------------------------------------
+
+class TestSetupGortexDisabled:
+    @patch("environments.gortex_docker.docker")
+    def test_setup_skips_gortex(self, mock_docker):
+        mock_client = MagicMock()
+        mock_container = MagicMock()
+        mock_container.short_id = "abc123"
+        mock_client.containers.run.return_value = mock_container
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        result = env.setup()
+        assert result is None
+        assert env._container is mock_container
+
+
+# -- setup container launch failure ------------------------------------------
+
+class TestSetupContainerFailure:
+    @patch("environments.gortex_docker.docker")
+    def test_container_launch_failure(self, mock_docker):
+        mock_client = MagicMock()
+        mock_client.containers.run.side_effect = RuntimeError("Docker not running")
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            instance_id="fail-instance",
+        )
+        result = env.setup()
+        assert result is not None
+        assert result["exit_status"] == "setup_failure"
+        assert "fail-instance" in result["instance_id"]

eval/tests/test_prompts_properties.py

@@ -0,0 +1,86 @@
+"""Property-based tests for eval.prompts module.
+
+Feature: eval-framework
+Uses hypothesis to verify prompt template loading and rendering properties.
+"""
+
+from __future__ import annotations
+
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from prompts import VALID_MODES, load_templates, render_instance_prompt
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+_mode_st = st.sampled_from(VALID_MODES)
+
+# Non-empty text strings for task descriptions.  Keep them printable and
+# reasonably sized so rendered output stays manageable.
+_task_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z")),
+    min_size=1,
+    max_size=200,
+)
+
+
+# ---------------------------------------------------------------------------
+# Property 5: Template loading consistency
+# Feature: eval-framework, Property 5: Template loading consistency
+# **Validates: Requirements 2.4, 11.1**
+# ---------------------------------------------------------------------------
+
+
+class TestTemplateLoadingConsistency:
+    """For any valid mode name, the loader returns matching
+    ``system_{mode}.jinja`` and ``instance_{mode}.jinja``."""
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_load_returns_two_templates(self, mode: str) -> None:
+        """load_templates always returns a 2-tuple for every valid mode."""
+        result = load_templates(mode)
+        assert isinstance(result, tuple)
+        assert len(result) == 2
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_system_template_matches_mode(self, mode: str) -> None:
+        """The system template's filename matches ``system_{mode}.jinja``."""
+        system_tpl, _ = load_templates(mode)
+        assert system_tpl.name == f"system_{mode}.jinja"
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_instance_template_matches_mode(self, mode: str) -> None:
+        """The instance template's filename matches ``instance_{mode}.jinja``."""
+        _, instance_tpl = load_templates(mode)
+        assert instance_tpl.name == f"instance_{mode}.jinja"
+
+
+# ---------------------------------------------------------------------------
+# Property 12: Template rendering includes task
+# Feature: eval-framework, Property 12: Template rendering includes task
+# **Validates: Requirements 11.2**
+# ---------------------------------------------------------------------------
+
+
+class TestTemplateRenderingIncludesTask:
+    """For any non-empty task string, rendered instance prompt contains the
+    task verbatim."""
+
+    @given(mode=_mode_st, task=_task_st)
+    @settings(max_examples=100)
+    def test_rendered_output_contains_task_verbatim(
+        self, mode: str, task: str
+    ) -> None:
+        """The rendered instance prompt must contain the task string as-is."""
+        _, instance_tpl = load_templates(mode)
+        rendered = render_instance_prompt(instance_tpl, task)
+        assert task in rendered, (
+            f"Task string not found verbatim in rendered output.\n"
+            f"  task:     {task!r}\n"
+            f"  rendered: {rendered[:300]!r}..."
+        )

eval/tests/test_results_properties.py

@@ -0,0 +1,293 @@
+"""Property-based tests for eval.results module.
+
+Feature: eval-framework
+Uses hypothesis to verify result completeness, serialization, and aggregation.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from results import InstanceResult, RunSummary, save_run_summary
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+_MODES = ["baseline", "native", "native_augment"]
+
+_TOOL_NAMES = [
+    "search_symbols",
+    "smart_context",
+    "explain_change_impact",
+    "graph_stats",
+    "find_usages",
+]
+
+_EXIT_STATUSES = ["submitted", "setup_failure", "api_error", "cost_limit", "step_limit"]
+
+
+def _gortex_tool_calls_st() -> st.SearchStrategy[dict[str, int]]:
+    """Strategy for gortex_metrics.tool_calls dict."""
+    return st.fixed_dictionaries(
+        {name: st.integers(min_value=0, max_value=50) for name in _TOOL_NAMES}
+    )
+
+
+def _gortex_metrics_st(mode: str) -> st.SearchStrategy[dict[str, Any]]:
+    """Strategy for gortex_metrics based on mode."""
+    if mode == "baseline":
+        return st.just({})
+    elif mode == "native":
+        return st.builds(
+            lambda tc: {
+                "tool_calls": tc,
+                "total_tool_calls": sum(tc.values()),
+            },
+            tc=_gortex_tool_calls_st(),
+        )
+    else:  # native_augment
+        return st.builds(
+            lambda tc, aug_calls, aug_hits, aug_errors, aug_time, idx_time: {
+                "tool_calls": tc,
+                "total_tool_calls": sum(tc.values()),
+                "augmentation_calls": aug_calls,
+                "augmentation_hits": aug_hits,
+                "augmentation_errors": aug_errors,
+                "augmentation_time_seconds": aug_time,
+                "index_time_seconds": idx_time,
+            },
+            tc=_gortex_tool_calls_st(),
+            aug_calls=st.integers(min_value=0, max_value=100),
+            aug_hits=st.integers(min_value=0, max_value=100),
+            aug_errors=st.integers(min_value=0, max_value=20),
+            aug_time=st.floats(min_value=0.0, max_value=60.0, allow_nan=False, allow_infinity=False),
+            idx_time=st.floats(min_value=0.0, max_value=300.0, allow_nan=False, allow_infinity=False),
+        )
+
+
+@st.composite
+def instance_result_st(draw: st.DrawFn, mode: str | None = None) -> InstanceResult:
+    """Strategy that generates a valid InstanceResult for a given mode."""
+    m = mode if mode is not None else draw(st.sampled_from(_MODES))
+    return InstanceResult(
+        instance_id=draw(st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=30,
+        )),
+        model=draw(st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        )),
+        mode=m,
+        exit_status=draw(st.sampled_from(_EXIT_STATUSES)),
+        submission=draw(st.text(min_size=0, max_size=200)),
+        cost=draw(st.floats(min_value=0.0, max_value=100.0, allow_nan=False, allow_infinity=False)),
+        tokens_input=draw(st.integers(min_value=0, max_value=500_000)),
+        tokens_output=draw(st.integers(min_value=0, max_value=100_000)),
+        n_calls=draw(st.integers(min_value=0, max_value=200)),
+        n_steps=draw(st.integers(min_value=0, max_value=100)),
+        duration_seconds=draw(st.floats(min_value=0.0, max_value=3600.0, allow_nan=False, allow_infinity=False)),
+        gortex_metrics=draw(_gortex_metrics_st(m)),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Property 7: Result completeness per mode
+# Feature: eval-framework, Property 7: Result completeness per mode
+# **Validates: Requirements 6.1, 6.2, 6.3**
+# ---------------------------------------------------------------------------
+
+
+class TestResultCompletenessPerMode:
+    """Base fields always present; native/native_augment include gortex tool
+    metrics; native_augment includes augmentation metrics."""
+
+    _BASE_FIELDS = {
+        "instance_id", "model", "mode", "exit_status", "submission",
+        "cost", "tokens_input", "tokens_output", "n_calls", "n_steps",
+        "duration_seconds", "gortex_metrics",
+    }
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_base_fields_always_present(self, result: InstanceResult) -> None:
+        """All base metric fields are present regardless of mode."""
+        d = result.to_dict()
+        for field_name in self._BASE_FIELDS:
+            assert field_name in d, f"Missing base field: {field_name}"
+
+    @given(result=instance_result_st(mode="native"))
+    @settings(max_examples=100)
+    def test_native_mode_has_gortex_tool_metrics(self, result: InstanceResult) -> None:
+        """Native mode results include gortex tool call metrics."""
+        metrics = result.gortex_metrics
+        assert "tool_calls" in metrics, "native mode must have tool_calls"
+        assert "total_tool_calls" in metrics, "native mode must have total_tool_calls"
+        for tool_name in _TOOL_NAMES:
+            assert tool_name in metrics["tool_calls"], f"Missing tool: {tool_name}"
+
+    @given(result=instance_result_st(mode="native_augment"))
+    @settings(max_examples=100)
+    def test_native_augment_mode_has_augmentation_metrics(self, result: InstanceResult) -> None:
+        """native_augment mode results include both tool and augmentation metrics."""
+        metrics = result.gortex_metrics
+        # Tool metrics
+        assert "tool_calls" in metrics
+        assert "total_tool_calls" in metrics
+        # Augmentation metrics
+        assert "augmentation_calls" in metrics, "native_augment must have augmentation_calls"
+        assert "augmentation_hits" in metrics, "native_augment must have augmentation_hits"
+        assert "augmentation_errors" in metrics, "native_augment must have augmentation_errors"
+        assert "augmentation_time_seconds" in metrics, "native_augment must have augmentation_time_seconds"
+
+    @given(result=instance_result_st(mode="baseline"))
+    @settings(max_examples=100)
+    def test_baseline_mode_has_empty_gortex_metrics(self, result: InstanceResult) -> None:
+        """Baseline mode results have empty gortex_metrics."""
+        assert result.gortex_metrics == {}
+
+
+# ---------------------------------------------------------------------------
+# Property 8: Result serialization round-trip
+# Feature: eval-framework, Property 8: Result serialization round-trip
+# **Validates: Requirements 6.4**
+# ---------------------------------------------------------------------------
+
+
+class TestResultSerializationRoundTrip:
+    """Serialize → deserialize produces equivalent object with all fields preserved."""
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_instance_result_round_trip(self, result: InstanceResult) -> None:
+        """InstanceResult survives to_dict → from_dict round-trip."""
+        d = result.to_dict()
+        restored = InstanceResult.from_dict(d)
+        assert restored.to_dict() == d
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_instance_result_json_round_trip(self, result: InstanceResult) -> None:
+        """InstanceResult survives to_dict → JSON string → parse → from_dict."""
+        d = result.to_dict()
+        json_str = json.dumps(d)
+        parsed = json.loads(json_str)
+        restored = InstanceResult.from_dict(parsed)
+        assert restored.to_dict() == d
+
+    @given(
+        run_id=st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        ),
+        model=st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        ),
+        mode=st.sampled_from(_MODES),
+    )
+    @settings(max_examples=100)
+    def test_run_summary_round_trip(self, run_id: str, model: str, mode: str) -> None:
+        """RunSummary survives to_dict → from_dict round-trip."""
+        summary = RunSummary(
+            run_id=run_id,
+            model=model,
+            mode=mode,
+            timestamp=1234567890.0,
+            config={"model": {"model_name": model}},
+            total_instances=10,
+            completed=8,
+            patch_rate=0.8,
+            total_cost=5.0,
+            mean_cost=0.5,
+            total_tokens=10000,
+            mean_tokens=1000.0,
+            total_duration_seconds=100.0,
+            mean_duration_seconds=10.0,
+        )
+        d = summary.to_dict()
+        restored = RunSummary.from_dict(d)
+        assert restored.to_dict() == d
+
+
+# ---------------------------------------------------------------------------
+# Property 9: Aggregate metric correctness
+# Feature: eval-framework, Property 9: Aggregate metric correctness
+# **Validates: Requirements 6.5, 7.1, 7.3**
+# ---------------------------------------------------------------------------
+
+
+class TestAggregateMetricCorrectness:
+    """patch_rate = patches/total, mean_cost = total_cost/count,
+    per-tool aggregations = sum of per-instance counts."""
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_patch_rate_equals_patches_over_total(self, results: list[InstanceResult]) -> None:
+        """patch_rate = count of results with non-empty submission / total."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        patches = sum(1 for r in results if r.submission)
+        expected_rate = patches / len(results)
+        assert abs(summary.patch_rate - expected_rate) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_cost_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_cost = total_cost / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_cost = sum(r.cost for r in results)
+        expected_mean = total_cost / len(results)
+        assert abs(summary.mean_cost - expected_mean) < 1e-9
+        assert abs(summary.total_cost - total_cost) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_tokens_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_tokens = total_tokens / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_tokens = sum(r.tokens_input + r.tokens_output for r in results)
+        expected_mean = total_tokens / len(results)
+        assert abs(summary.total_tokens - total_tokens) < 1e-9
+        assert abs(summary.mean_tokens - expected_mean) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_duration_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_duration = total_duration / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_duration = sum(r.duration_seconds for r in results)
+        expected_mean = total_duration / len(results)
+        assert abs(summary.total_duration_seconds - total_duration) < 1e-9
+        assert abs(summary.mean_duration_seconds - expected_mean) < 1e-9

eval/tests/test_runner_properties.py

@@ -0,0 +1,375 @@
+"""Property-based tests for eval runner (run_eval module).
+
+Feature: eval-framework
+Uses hypothesis to verify runner orchestration properties.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+from unittest.mock import patch, MagicMock
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from run_eval import parse_slice, build_matrix_configs, run_configuration
+
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Bounded integers suitable for slice components.
+_slice_int_st = st.integers(min_value=-50, max_value=50)
+
+# Optional slice int (None means omitted).
+_opt_slice_int_st = st.one_of(st.none(), _slice_int_st)
+
+# Simple identifier-like strings for model/mode names.
+_name_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+    min_size=1,
+    max_size=12,
+)
+
+# Non-empty lists of unique names.
+_name_list_st = st.lists(_name_st, min_size=1, max_size=6, unique=True)
+
+
+def _make_instance(instance_id: str) -> Dict[str, Any]:
+    """Create a minimal fake SWE-bench instance dict."""
+    return {
+        "instance_id": instance_id,
+        "problem_statement": f"Fix {instance_id}",
+    }
+
+
+
+# ---------------------------------------------------------------------------
+# Property 3: Slice parsing correctness
+# Feature: eval-framework, Property 3: Slice parsing correctness
+# **Validates: Requirements 1.5**
+# ---------------------------------------------------------------------------
+
+
+class TestSliceParsingCorrectness:
+    """For any valid slice spec, result matches Python's list[start:end] semantics."""
+
+    @given(start=_opt_slice_int_st, end=_opt_slice_int_st)
+    @settings(max_examples=100)
+    def test_two_part_slice_matches_python(
+        self, start: int | None, end: int | None
+    ) -> None:
+        """A 'start:end' spec produces the same sublist as list[start:end]."""
+        # Build the spec string.
+        start_str = "" if start is None else str(start)
+        end_str = "" if end is None else str(end)
+        spec = f"{start_str}:{end_str}"
+
+        # Reference list large enough to exercise the slice.
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[start:end]
+
+    @given(start=_opt_slice_int_st, end=_opt_slice_int_st, step=_slice_int_st)
+    @settings(max_examples=100)
+    def test_three_part_slice_matches_python(
+        self, start: int | None, end: int | None, step: int
+    ) -> None:
+        """A 'start:end:step' spec produces the same sublist as list[start:end:step]."""
+        assume(step != 0)  # step=0 is invalid for Python slices
+
+        start_str = "" if start is None else str(start)
+        end_str = "" if end is None else str(end)
+        spec = f"{start_str}:{end_str}:{step}"
+
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[start:end:step]
+
+    @given(end=st.integers(min_value=-50, max_value=50))
+    @settings(max_examples=100)
+    def test_single_value_treated_as_end(self, end: int) -> None:
+        """A single integer spec like '5' is treated as slice(None, 5)."""
+        spec = str(end)
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[:end]
+
+    def test_empty_spec_selects_everything(self) -> None:
+        """An empty string selects all elements."""
+        ref = list(range(20))
+        parsed = parse_slice("")
+        assert ref[parsed] == ref
+
+
+
+# ---------------------------------------------------------------------------
+# Property 1: Instance execution completeness
+# Feature: eval-framework, Property 1: Instance execution completeness
+# **Validates: Requirements 1.1**
+# ---------------------------------------------------------------------------
+
+
+def _fake_process_instance(instance, config, output_dir, run_id, model_name, mode_name):
+    """A mock process_instance that returns a result dict without side effects."""
+    return {
+        "instance_id": instance["instance_id"],
+        "model": model_name,
+        "mode": mode_name,
+        "exit_status": "submitted",
+        "submission": "fake-patch",
+        "cost": 0.01,
+    }
+
+
+class TestInstanceExecutionCompleteness:
+    """For any N instances and worker count W >= 1, runner produces exactly N
+    result records."""
+
+    @given(
+        n=st.integers(min_value=1, max_value=20),
+        workers=st.integers(min_value=1, max_value=4),
+    )
+    @settings(max_examples=100)
+    def test_produces_exactly_n_results(self, n: int, workers: int) -> None:
+        """run_configuration returns exactly N results for N instances."""
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_fake_process_instance),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers
+            )
+
+        assert len(results) == n
+
+    @given(n=st.integers(min_value=1, max_value=15))
+    @settings(max_examples=100)
+    def test_each_instance_id_present(self, n: int) -> None:
+        """Every instance ID appears exactly once in the results."""
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_fake_process_instance),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), 1
+            )
+
+        result_ids = [r["instance_id"] for r in results]
+        expected_ids = [inst["instance_id"] for inst in instances]
+        assert sorted(result_ids) == sorted(expected_ids)
+
+
+
+# ---------------------------------------------------------------------------
+# Property 2: Matrix cross-product completeness
+# Feature: eval-framework, Property 2: Matrix cross-product completeness
+# **Validates: Requirements 1.3**
+# ---------------------------------------------------------------------------
+
+
+class TestMatrixCrossProductCompleteness:
+    """For M models and K modes, matrix produces exactly M x K unique
+    (model, mode) configs."""
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_produces_m_times_k_configs(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """build_matrix_configs returns exactly M * K pairs."""
+        configs = build_matrix_configs(models, modes)
+        assert len(configs) == len(models) * len(modes)
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_all_pairs_unique(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """Every (model, mode) pair in the result is unique."""
+        configs = build_matrix_configs(models, modes)
+        assert len(set(configs)) == len(configs)
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_every_model_mode_combination_present(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """Every possible (model, mode) combination appears in the result."""
+        configs = build_matrix_configs(models, modes)
+        config_set = set(configs)
+        for model in models:
+            for mode in modes:
+                assert (model, mode) in config_set
+
+
+
+# ---------------------------------------------------------------------------
+# Property 4: Failure isolation
+# Feature: eval-framework, Property 4: Failure isolation
+# **Validates: Requirements 1.7**
+# ---------------------------------------------------------------------------
+
+
+class TestFailureIsolation:
+    """For N instances where K fail, runner still produces results for all
+    N-K non-failing instances plus K failure entries.
+
+    The parallel path (workers >= 2) in run_configuration catches exceptions
+    from process_instance and records them as error entries. We test with
+    workers=2 to exercise this failure isolation logic.
+    """
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_failure_isolation_produces_n_results(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Even when K instances fail, we get exactly N total result records."""
+        k = data.draw(st.integers(min_value=0, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        assert len(results) == n
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_non_failing_instances_have_results(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Non-failing instances produce normal result records."""
+        k = data.draw(st.integers(min_value=1, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        # Non-failing instances should have "submitted" status.
+        non_failing_results = [
+            r for r in results if r["instance_id"] not in failing_ids
+        ]
+        assert len(non_failing_results) == n - k
+        for r in non_failing_results:
+            assert r["exit_status"] == "submitted"
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_failing_instances_recorded_as_errors(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Failing instances are recorded with error status."""
+        k = data.draw(st.integers(min_value=1, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        # Failing instances should have "error" status.
+        error_results = [
+            r for r in results if r["instance_id"] in failing_ids
+        ]
+        assert len(error_results) == k
+        for r in error_results:
+            assert r["exit_status"] == "error"

eval/tests/test_smoke.py

@@ -0,0 +1,130 @@
+"""Smoke tests for the eval framework.
+
+Feature: eval-framework
+Verifies basic sanity of all major components:
+- Bridge scripts pass bash -n
+- All prompt templates render with sample data
+- list_configs discovers YAML files
+- gortex eval-server --help exits 0 (skipped if binary unavailable)
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from config import list_configs
+from prompts import VALID_MODES, load_templates, render_instance_prompt
+
+# --- Paths ---
+EVAL_DIR = Path(__file__).resolve().parent.parent
+BRIDGE_DIR = EVAL_DIR / "bridge"
+PROMPTS_DIR = EVAL_DIR / "prompts"
+
+
+# --- Bridge script bash -n tests ---
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+) if BRIDGE_DIR.is_dir() else []
+
+
+@pytest.mark.parametrize("script", BRIDGE_SCRIPTS, ids=lambda p: p.name)
+def test_bridge_script_bash_syntax(script: Path) -> None:
+    """Each bridge script must pass bash -n (syntax check)."""
+    result = subprocess.run(
+        ["bash", "-n", str(script)],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"bash -n failed for {script.name}: {result.stderr}"
+    )
+
+
+# --- Prompt template rendering tests ---
+
+SAMPLE_TASK = (
+    "Fix the bug in django/contrib/auth/models.py where "
+    "AbstractUser.clean() does not normalize the email address."
+)
+
+
+@pytest.mark.parametrize("mode", VALID_MODES)
+def test_prompt_templates_render_without_errors(mode: str) -> None:
+    """All prompt templates render without errors with sample data."""
+    system_tpl, instance_tpl = load_templates(mode)
+
+    # System template should render (may have no variables)
+    system_output = system_tpl.render()
+    assert len(system_output) > 0, f"system_{mode}.jinja rendered empty"
+
+    # Instance template should render with task variable
+    instance_output = render_instance_prompt(instance_tpl, SAMPLE_TASK)
+    assert len(instance_output) > 0, f"instance_{mode}.jinja rendered empty"
+    assert SAMPLE_TASK in instance_output, (
+        f"instance_{mode}.jinja did not include the task verbatim"
+    )
+
+
+# --- list_configs discovery tests ---
+
+def test_list_configs_discovers_yaml_files() -> None:
+    """list_configs discovers all YAML files in configs/."""
+    configs = list_configs()
+
+    assert "models" in configs
+    assert "modes" in configs
+
+    # We know at least these configs exist
+    assert len(configs["models"]) >= 2, (
+        f"Expected at least 2 model configs, found: {configs['models']}"
+    )
+    assert len(configs["modes"]) >= 3, (
+        f"Expected at least 3 mode configs, found: {configs['modes']}"
+    )
+
+    # Verify known configs are present
+    assert "claude-sonnet" in configs["models"]
+    assert "claude-haiku" in configs["models"]
+    assert "baseline" in configs["modes"]
+    assert "native" in configs["modes"]
+    assert "native_augment" in configs["modes"]
+
+
+# --- gortex eval-server --help test ---
+
+# Check if gortex binary is available
+_gortex_binary = shutil.which("gortex") or (
+    str(EVAL_DIR.parent / "gortex") if (EVAL_DIR.parent / "gortex").is_file() else None
+)
+
+
+@pytest.mark.skipif(
+    _gortex_binary is None,
+    reason="gortex binary not found in PATH or project root",
+)
+def test_gortex_eval_server_help_exits_zero() -> None:
+    """gortex eval-server --help exits 0."""
+    result = subprocess.run(
+        [_gortex_binary, "eval-server", "--help"],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"gortex eval-server --help failed:\n"
+        f"stdout: {result.stdout}\n"
+        f"stderr: {result.stderr}"
+    )
+    assert "eval-server" in result.stdout.lower() or "eval-server" in result.stderr.lower(), (
+        "Help output should mention eval-server"
+    )

eval/utils/__init__.py

@@ -0,0 +1 @@
+"""Shared utilities for the eval framework."""