chore: import upstream snapshot with attribution

eval/agents/gortex_agent.py

@@ -0,0 +1,347 @@
+"""Gortex-Enhanced Agent for SWE-bench Evaluation.
+
+Extends mini-swe-agent's DefaultAgent with Gortex code intelligence:
+
+1. **baseline** — bash only (grep, find, cat, sed). Control group.
+2. **native** — bash + Gortex tool bridge scripts via eval-server.
+3. **native_augment** — native + automatic grep output augmentation
+   with ``[Gortex]`` graph annotations (recommended).
+
+The agent is designed to work standalone even when mini-swe-agent is
+not installed — a lightweight base class is used as a fallback.
+
+Heavy lifting lives elsewhere:
+- Prompt selection: ``eval.prompts`` (system + instance templates per mode)
+- Augmentation pipeline: ``eval.augmentation`` (task 12)
+- Metrics persistence: ``eval.results`` (task 15)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger("gortex_agent")
+
+# ---------------------------------------------------------------------------
+# Try to import mini-swe-agent; fall back to a lightweight stub.
+# ---------------------------------------------------------------------------
+try:
+    from minisweagent.agents.default import DefaultAgent as _DefaultAgent
+except ImportError:  # pragma: no cover
+    class _DefaultAgent:  # type: ignore[no-redef]
+        """Minimal stand-in when mini-swe-agent is not installed."""
+
+        def __init__(self, **kwargs: Any) -> None:
+            self._kwargs = kwargs
+            self._step_count = 0
+
+        def run(self, task: str) -> dict:
+            raise NotImplementedError(
+                "mini-swe-agent is not installed — "
+                "install it or override run() in a subclass"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Gortex evaluation modes
+# ---------------------------------------------------------------------------
+
+class GortexMode(str, Enum):
+    """Evaluation modes for Gortex integration."""
+
+    BASELINE = "baseline"
+    NATIVE = "native"
+    NATIVE_AUGMENT = "native_augment"
+
+
+# ---------------------------------------------------------------------------
+# Tool bridge binaries → metric keys
+# ---------------------------------------------------------------------------
+
+# Maps a short metric key to the bash binary name installed in the container.
+TOOL_BINARIES: Dict[str, str] = {
+    "search_symbols": "gortex-search",
+    "smart_context": "gortex-context",
+    "explain_change_impact": "gortex-impact",
+    "graph_stats": "gortex-overview",
+    "find_usages": "gortex-usages",
+    "augment": "gortex-augment",
+}
+
+TOOL_METRIC_KEYS: List[str] = list(TOOL_BINARIES.keys())
+
+
+# ---------------------------------------------------------------------------
+# Metrics
+# ---------------------------------------------------------------------------
+
+@dataclass
+class GortexMetrics:
+    """Tracks Gortex-specific metrics during an evaluation run."""
+
+    tool_calls: Dict[str, int] = field(default_factory=lambda: {k: 0 for k in TOOL_METRIC_KEYS})
+    augmentation_calls: int = 0
+    augmentation_hits: int = 0
+    augmentation_errors: int = 0
+    augmentation_time_seconds: float = 0.0
+
+    @property
+    def total_tool_calls(self) -> int:
+        return sum(self.tool_calls.values())
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "tool_calls": dict(self.tool_calls),
+            "total_tool_calls": self.total_tool_calls,
+            "augmentation_calls": self.augmentation_calls,
+            "augmentation_hits": self.augmentation_hits,
+            "augmentation_errors": self.augmentation_errors,
+            "augmentation_time_seconds": round(self.augmentation_time_seconds, 2),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Pattern extraction (shared with augmentation pipeline)
+# ---------------------------------------------------------------------------
+
+_GREP_PATTERNS = [
+    # Quoted pattern: grep -rn "pattern" .
+    re.compile(r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*["\']([^"\']+)["\']'),
+    # Unquoted pattern: grep -rn pattern .
+    re.compile(r'(?:grep|rg|ag)\s+(?:-[a-zA-Z]*\s+)*(\S+)'),
+]
+
+
+def extract_search_pattern(command: str) -> Optional[str]:
+    """Extract the search pattern from a grep/rg/ag command string.
+
+    Returns ``None`` when no usable pattern can be identified.
+    """
+    for pat in _GREP_PATTERNS:
+        match = pat.search(command)
+        if match:
+            result = match.group(1)
+            # Skip file paths and flags that were mis-captured
+            if result.startswith(("/", ".", "-")):
+                continue
+            return result
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Agent
+# ---------------------------------------------------------------------------
+
+class GortexAgent(_DefaultAgent):
+    """LLM agent with optional Gortex code-intelligence augmentation.
+
+    In **baseline** mode the agent behaves like a plain ``DefaultAgent``
+    with standard bash tools only.
+
+    In **native** mode the Gortex tool bridge scripts are available as
+    additional bash commands (``gortex-search``, ``gortex-context``, …).
+
+    In **native_augment** mode the agent additionally intercepts grep/rg
+    output and enriches it with ``[Gortex]`` graph annotations.
+
+    Parameters
+    ----------
+    config:
+        Merged run configuration dict (model + mode YAML).  The agent
+        reads ``config["agent"]`` for its own settings.
+    """
+
+    def __init__(self, config: Dict[str, Any], model: Any = None, env: Any = None, **kwargs: Any) -> None:
+        if model is not None and env is not None:
+            super().__init__(model=model, env=env, **kwargs)
+        else:
+            # Standalone mode (no mini-swe-agent)
+            self._kwargs = kwargs
+            self._step_count = 0
+
+        agent_cfg = config.get("agent", {})
+
+        # --- mode -----------------------------------------------------------
+        raw_mode = agent_cfg.get("gortex_mode", "baseline")
+        if isinstance(raw_mode, GortexMode):
+            self.mode = raw_mode
+        else:
+            self.mode = GortexMode(raw_mode)
+
+        # --- limits ----------------------------------------------------------
+        self.cost_limit: float = float(agent_cfg.get("cost_limit", 3.0))
+        self.step_limit: int = int(agent_cfg.get("step_limit", 30))
+
+        # --- augmentation settings -------------------------------------------
+        self.augment_timeout: float = float(agent_cfg.get("augment_timeout", 5.0))
+        self.augment_min_pattern_length: int = int(
+            agent_cfg.get("augment_min_pattern_length", 3)
+        )
+        self.track_gortex_usage: bool = bool(
+            agent_cfg.get("track_gortex_usage", True)
+        )
+
+        # --- prompt templates ------------------------------------------------
+        self._system_template = None
+        self._instance_template = None
+        self._load_prompt_templates()
+
+        # --- metrics ---------------------------------------------------------
+        self.metrics = GortexMetrics()
+
+        # --- internal bookkeeping --------------------------------------------
+        self._step_count = 0
+        self._total_cost: float = 0.0
+
+        logger.info(
+            "GortexAgent initialised: mode=%s, step_limit=%d, cost_limit=%.2f",
+            self.mode.value,
+            self.step_limit,
+            self.cost_limit,
+        )
+
+    # ------------------------------------------------------------------
+    # Prompt loading
+    # ------------------------------------------------------------------
+
+    def _load_prompt_templates(self) -> None:
+        """Load mode-specific Jinja2 prompt templates via ``eval.prompts``."""
+        try:
+            from prompts import load_templates
+
+            self._system_template, self._instance_template = load_templates(
+                self.mode.value
+            )
+            logger.debug("Loaded prompt templates for mode=%s", self.mode.value)
+        except Exception as exc:
+            logger.warning("Could not load prompt templates: %s", exc)
+
+    def render_system_prompt(self) -> str:
+        """Render the system prompt for the current mode."""
+        if self._system_template is None:
+            return ""
+        return self._system_template.render()
+
+    def render_instance_prompt(self, task: str) -> str:
+        """Render the instance prompt with the given *task* description."""
+        if self._instance_template is None:
+            return task
+        try:
+            from prompts import render_instance_prompt
+
+            return render_instance_prompt(self._instance_template, task)
+        except Exception:
+            return task
+
+    # ------------------------------------------------------------------
+    # Execution helpers
+    # ------------------------------------------------------------------
+
+    def should_continue(self) -> bool:
+        """Return ``False`` when a cost or step limit has been reached."""
+        if self._step_count >= self.step_limit:
+            logger.info("Step limit reached (%d)", self.step_limit)
+            return False
+        if self._total_cost >= self.cost_limit:
+            logger.info("Cost limit reached ($%.2f)", self.cost_limit)
+            return False
+        return True
+
+    def record_step(self, cost: float = 0.0) -> None:
+        """Record one agent step and its associated API cost."""
+        self._step_count += 1
+        self._total_cost += cost
+
+    # ------------------------------------------------------------------
+    # Tool-usage tracking
+    # ------------------------------------------------------------------
+
+    def track_tool_usage(self, command: str) -> None:
+        """Inspect *command* and increment the matching tool-call counter."""
+        if not self.track_gortex_usage:
+            return
+        for key, binary in TOOL_BINARIES.items():
+            if binary in command:
+                self.metrics.tool_calls[key] = self.metrics.tool_calls.get(key, 0) + 1
+                break
+
+    # ------------------------------------------------------------------
+    # Grep augmentation (native_augment mode)
+    # ------------------------------------------------------------------
+
+    def maybe_augment(
+        self,
+        command: str,
+        output: str,
+        *,
+        execute_fn: Any = None,
+    ) -> str:
+        """Conditionally augment grep/rg output with Gortex annotations.
+
+        In ``native_augment`` mode, if *command* is a grep/rg invocation
+        with a pattern of sufficient length, the augmentation endpoint is
+        called and ``[Gortex]`` annotations are appended.
+
+        Parameters
+        ----------
+        command:
+            The bash command that was executed.
+        output:
+            The raw stdout captured from the command.
+        execute_fn:
+            A callable ``(cmd: str, timeout: float) -> str`` that runs a
+            command inside the container and returns its stdout.  When
+            ``None``, augmentation is skipped.
+
+        Returns
+        -------
+        str
+            The (possibly enriched) output.
+        """
+        if self.mode != GortexMode.NATIVE_AUGMENT:
+            return output
+
+        pattern = extract_search_pattern(command)
+        if not pattern or len(pattern) < self.augment_min_pattern_length:
+            return output
+
+        if execute_fn is None:
+            return output
+
+        start = time.time()
+        try:
+            augment_result = execute_fn(
+                f'gortex-augment "{pattern}" 2>&1 || true',
+                self.augment_timeout,
+            )
+            elapsed = time.time() - start
+            self.metrics.augmentation_calls += 1
+            self.metrics.augmentation_time_seconds += elapsed
+
+            augment_text = (augment_result or "").strip()
+            if augment_text and "[Gortex]" in augment_text:
+                self.metrics.augmentation_hits += 1
+                return f"{output}\n\n{augment_text}"
+        except Exception as exc:
+            logger.debug("Augmentation failed for pattern '%s': %s", pattern, exc)
+            self.metrics.augmentation_errors += 1
+
+        return output
+
+    # ------------------------------------------------------------------
+    # Serialization
+    # ------------------------------------------------------------------
+
+    def get_metrics(self) -> Dict[str, Any]:
+        """Return a dict of Gortex-specific metrics for result storage."""
+        return {
+            "mode": self.mode.value,
+            "step_count": self._step_count,
+            "total_cost": round(self._total_cost, 4),
+            "gortex_metrics": self.metrics.to_dict(),
+        }