chore: import upstream snapshot with attribution

eval/tests/__init__.py

@@ -0,0 +1 @@
+"""Eval framework test suite."""

eval/tests/test_augmentation_properties.py

@@ -0,0 +1,191 @@
+"""Property-based tests for eval.augmentation module.
+
+Feature: eval-framework
+Uses hypothesis to verify augmentation triggering and timeout properties.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict
+from unittest.mock import MagicMock, patch
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from augmentation import augment_grep_output
+
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Printable text that could appear in grep output lines.
+_grep_line_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z"), whitelist_characters=(":", "/", ".", "_", "-")),
+    min_size=1,
+    max_size=80,
+)
+
+# Multi-line grep output.
+_grep_output_st = st.lists(_grep_line_st, min_size=1, max_size=10).map("\n".join)
+
+# Search patterns: printable, no quotes, no leading special chars.
+_pattern_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_",)),
+    min_size=1,
+    max_size=30,
+).filter(lambda s: not s.startswith(("/", ".", "-")))
+
+# Minimum pattern length config values.
+_min_len_st = st.integers(min_value=1, max_value=20)
+
+
+def _make_grep_command(pattern: str) -> str:
+    """Build a grep command string with the given pattern."""
+    return f'grep -rn "{pattern}" .'
+
+
+# ---------------------------------------------------------------------------
+# Property 13: Augmentation triggering rules
+# Feature: eval-framework, Property 13: Augmentation triggering rules
+# **Validates: Requirements 9.1, 9.4**
+# ---------------------------------------------------------------------------
+
+
+class TestAugmentationTriggeringRules:
+    """For any grep/rg command in native_augment mode, if the extracted search
+    pattern has length >= the configured minimum, augmentation SHALL be
+    attempted. If the pattern length is below the minimum, augmentation SHALL
+    be skipped and the original output returned."""
+
+    @given(pattern=_pattern_st, min_len=_min_len_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_short_pattern_skips_augmentation(
+        self, pattern: str, min_len: int, raw_output: str
+    ) -> None:
+        """Pattern shorter than minimum → augmentation skipped, original output returned."""
+        assume(len(pattern) < min_len)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": min_len}
+
+        # urlopen should never be called when pattern is too short.
+        with patch("augmentation.urllib.request.urlopen") as mock_urlopen:
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        mock_urlopen.assert_not_called()
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, min_len=_min_len_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_long_pattern_attempts_augmentation(
+        self, pattern: str, min_len: int, raw_output: str
+    ) -> None:
+        """Pattern >= minimum length → augmentation attempted (HTTP call made)."""
+        assume(len(pattern) >= min_len)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": min_len}
+
+        # Mock urlopen to return a valid augmentation response.
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({
+            "callers": [{"name": "Caller", "location": "file.go:1"}],
+        }).encode("utf-8")
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+
+        with patch("augmentation.urllib.request.urlopen", return_value=mock_response) as mock_urlopen:
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        mock_urlopen.assert_called_once()
+        # Result should contain the Gortex annotation.
+        assert "[Gortex]" in result
+
+
+# ---------------------------------------------------------------------------
+# Property 14: Augmentation timeout preserves output
+# Feature: eval-framework, Property 14: Augmentation timeout preserves output
+# **Validates: Requirements 9.3**
+# ---------------------------------------------------------------------------
+
+
+class TestAugmentationTimeoutPreservesOutput:
+    """For any grep output where augmentation times out or returns nothing,
+    returned output is identical to original."""
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_timeout_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When the augmentation endpoint times out, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            side_effect=TimeoutError("timed out"),
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_connection_error_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When the augmentation endpoint is unreachable, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        import urllib.error
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            side_effect=urllib.error.URLError("connection refused"),
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output
+
+    @given(pattern=_pattern_st, raw_output=_grep_output_st)
+    @settings(max_examples=100)
+    def test_empty_response_returns_original(
+        self, pattern: str, raw_output: str
+    ) -> None:
+        """When augmentation returns no useful context, original output is returned."""
+        assume(len(pattern) >= 3)
+
+        command = _make_grep_command(pattern)
+        config: Dict[str, Any] = {"augment_min_pattern_length": 1}
+
+        # Return empty annotations (no callers/callees/flows).
+        mock_response = MagicMock()
+        mock_response.read.return_value = json.dumps({}).encode("utf-8")
+        mock_response.__enter__ = MagicMock(return_value=mock_response)
+        mock_response.__exit__ = MagicMock(return_value=False)
+
+        with patch(
+            "eval.augmentation.urllib.request.urlopen",
+            return_value=mock_response,
+        ):
+            result = augment_grep_output(
+                raw_output, command, "http://127.0.0.1:4747", config
+            )
+
+        assert result == raw_output

eval/tests/test_bridge_properties.py

@@ -0,0 +1,210 @@
+"""Property-based tests for tool bridge output format.
+
+Feature: eval-framework, Property 15: Tool bridge output format
+**Validates: Requirements 8.4, 8.5**
+
+For any valid Gortex tool JSON response, formatted output is plain text
+(not raw JSON) and contains next-step hints guiding the agent toward
+effective tool chaining.
+
+Since bridge scripts require a running eval-server for full execution,
+we test the formatting properties by:
+1. Verifying scripts contain "Next steps" hints in their output
+2. Piping mock JSON through the jq formatting logic extracted from the scripts
+3. Using hypothesis to generate various JSON response shapes
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import shutil
+from pathlib import Path
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+BRIDGE_DIR = Path(__file__).resolve().parent.parent / "bridge"
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+)
+
+# User-facing scripts (gortex-augment is an internal helper without next-step hints)
+USER_FACING_SCRIPTS = [s for s in BRIDGE_SCRIPTS if s.name != "gortex-augment"]
+
+# The jq filter used by bridge scripts to format MCP responses.
+# Must check `type` before accessing keys to avoid "Cannot index array" errors.
+JQ_FORMAT_FILTER = r'''
+if type == "array" then
+    .[] | if .id then "\(.id)  \(.kind // "")  \(.file // "")" else tostring end
+elif type == "object" and .content then
+    .content[] | select(.type == "text") | .text
+elif type == "object" and .error then
+    "Error: \(.error)"
+else
+    tostring
+end
+'''
+
+HAS_JQ = shutil.which("jq") is not None
+
+
+# ---------------------------------------------------------------------------
+# Strategies for generating Gortex-like JSON responses
+# ---------------------------------------------------------------------------
+
+_safe_text = st.text(
+    alphabet=st.characters(
+        whitelist_categories=("L", "N", "P", "Z"),
+        blacklist_characters=("\x00", "{", "["),
+    ),
+    min_size=1,
+    max_size=100,
+).filter(lambda s: s.strip())
+
+# MCP-style content response: {"content": [{"type": "text", "text": "..."}]}
+_mcp_content_st = st.builds(
+    lambda texts: {"content": [{"type": "text", "text": t} for t in texts]},
+    texts=st.lists(_safe_text, min_size=1, max_size=3),
+)
+
+# Array of symbol-like objects
+_symbol_obj_st = st.fixed_dictionaries({
+    "id": _safe_text,
+    "kind": st.sampled_from(["function", "method", "type", "interface", "variable"]),
+    "file": _safe_text,
+})
+
+_symbol_array_st = st.lists(_symbol_obj_st, min_size=1, max_size=5)
+
+# Error response
+_error_response_st = st.builds(
+    lambda msg: {"error": msg},
+    msg=_safe_text,
+)
+
+# Combined strategy for any valid Gortex response shape
+_gortex_response_st = st.one_of(
+    _mcp_content_st,
+    _symbol_array_st,
+    _error_response_st,
+)
+
+
+def _run_jq(json_input: str, jq_filter: str) -> subprocess.CompletedProcess:
+    """Run jq with the given filter on the input JSON string."""
+    return subprocess.run(
+        ["jq", "-r", jq_filter],
+        input=json_input,
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Property 15: Tool bridge output format
+# ---------------------------------------------------------------------------
+
+
+class TestToolBridgeOutputFormat:
+    """For any valid Gortex tool JSON response, formatted output is plain text
+    (not raw JSON) and contains next-step hints."""
+
+    def test_all_scripts_contain_next_steps_section(self) -> None:
+        """Every user-facing bridge script must include a 'Next steps' section."""
+        for script in USER_FACING_SCRIPTS:
+            content = script.read_text()
+            assert "Next steps" in content, (
+                f"{script.name} does not contain 'Next steps' hints"
+            )
+
+    def test_all_scripts_contain_gortex_tool_hints(self) -> None:
+        """Next-step hints should reference other gortex-* tools for chaining."""
+        for script in USER_FACING_SCRIPTS:
+            content = script.read_text()
+            # Each script should suggest at least one other gortex-* tool
+            hint_tools = [
+                "gortex-search", "gortex-context", "gortex-impact",
+                "gortex-overview", "gortex-usages", "gortex-augment",
+            ]
+            other_tools = [t for t in hint_tools if t != script.name]
+            has_hint = any(tool in content for tool in other_tools)
+            assert has_hint, (
+                f"{script.name} does not reference any other gortex-* tools in hints"
+            )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_mcp_content_st)
+    @settings(max_examples=100, deadline=None)
+    def test_mcp_content_formatted_as_plain_text(self, response: dict) -> None:
+        """MCP content responses are formatted as plain text, not raw JSON."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, "Formatted output should not be empty"
+
+        # Output should NOT look like raw JSON (no leading { or [)
+        assert not output.startswith("{"), (
+            f"Output looks like raw JSON object: {output[:80]}"
+        )
+        assert not output.startswith("["), (
+            f"Output looks like raw JSON array: {output[:80]}"
+        )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(symbols=_symbol_array_st)
+    @settings(max_examples=100, deadline=None)
+    def test_symbol_array_formatted_as_plain_text(self, symbols: list) -> None:
+        """Symbol array responses are formatted as readable lines, not JSON."""
+        json_str = json.dumps(symbols)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, "Formatted output should not be empty"
+
+        # Each symbol should produce a line with its id
+        lines = output.split("\n")
+        assert len(lines) >= 1, "Expected at least one output line per symbol"
+
+        # Output should not be raw JSON
+        for line in lines:
+            stripped = line.strip()
+            if stripped:
+                assert not stripped.startswith("{"), (
+                    f"Line looks like raw JSON: {stripped[:80]}"
+                )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_error_response_st)
+    @settings(max_examples=100, deadline=None)
+    def test_error_response_formatted_as_plain_text(self, response: dict) -> None:
+        """Error responses are formatted as 'Error: ...' text, not raw JSON."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert output.startswith("Error:"), (
+            f"Error response should start with 'Error:' but got: {output[:80]}"
+        )
+
+    @pytest.mark.skipif(not HAS_JQ, reason="jq not installed")
+    @given(response=_gortex_response_st)
+    @settings(max_examples=100, deadline=None)
+    def test_any_response_produces_non_empty_output(self, response) -> None:
+        """Any valid Gortex response shape produces non-empty formatted output."""
+        json_str = json.dumps(response)
+        result = _run_jq(json_str, JQ_FORMAT_FILTER)
+
+        assert result.returncode == 0, f"jq failed: {result.stderr}"
+        output = result.stdout.strip()
+        assert len(output) > 0, (
+            f"Expected non-empty output for response: {json_str[:120]}"
+        )

eval/tests/test_bridge_syntax.py

@@ -0,0 +1,48 @@
+"""Bash syntax validation tests for tool bridge scripts.
+
+Feature: eval-framework
+Verifies all bridge scripts in eval/bridge/ pass `bash -n` (syntax check).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+# All bridge scripts in eval/bridge/ (no .py files, no __pycache__)
+BRIDGE_DIR = Path(__file__).resolve().parent.parent / "bridge"
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+)
+
+
+@pytest.fixture(params=BRIDGE_SCRIPTS, ids=lambda p: p.name)
+def bridge_script(request: pytest.FixtureRequest) -> Path:
+    return request.param
+
+
+def test_bridge_scripts_discovered() -> None:
+    """Sanity check: we found at least the 6 expected bridge scripts."""
+    assert len(BRIDGE_SCRIPTS) >= 6, (
+        f"Expected at least 6 bridge scripts, found {len(BRIDGE_SCRIPTS)}: "
+        f"{[p.name for p in BRIDGE_SCRIPTS]}"
+    )
+
+
+def test_bash_syntax_valid(bridge_script: Path) -> None:
+    """Each bridge script must pass bash -n (syntax check)."""
+    result = subprocess.run(
+        ["bash", "-n", str(bridge_script)],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"bash -n failed for {bridge_script.name}:\n"
+        f"stderr: {result.stderr}\n"
+        f"stdout: {result.stdout}"
+    )

eval/tests/test_config.py

@@ -0,0 +1,200 @@
+"""Unit tests for eval.config module."""
+
+from __future__ import annotations
+
+import textwrap
+from pathlib import Path
+
+import pytest
+import yaml
+
+from config import (
+    list_configs,
+    load_mode_config,
+    load_model_config,
+    merge_configs,
+    validate_config,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def tmp_configs(tmp_path, monkeypatch):
+    """Create a temporary configs directory and patch _CONFIGS_DIR."""
+    models_dir = tmp_path / "models"
+    modes_dir = tmp_path / "modes"
+    models_dir.mkdir()
+    modes_dir.mkdir()
+
+    import config as config_mod
+
+    monkeypatch.setattr(config_mod, "_CONFIGS_DIR", tmp_path)
+    return tmp_path
+
+
+def _write_yaml(path: Path, data: dict) -> None:
+    with open(path, "w") as f:
+        yaml.safe_dump(data, f)
+
+
+# ---------------------------------------------------------------------------
+# load_model_config
+# ---------------------------------------------------------------------------
+
+
+class TestLoadModelConfig:
+    def test_loads_valid_yaml(self, tmp_configs):
+        data = {"model": {"model_name": "test-model", "cost_tracking": "ignore_errors"}}
+        _write_yaml(tmp_configs / "models" / "test.yaml", data)
+        result = load_model_config("test")
+        assert result == data
+
+    def test_missing_config_raises(self, tmp_configs):
+        with pytest.raises(FileNotFoundError, match="Model config not found"):
+            load_model_config("nonexistent")
+
+    def test_empty_yaml_returns_empty_dict(self, tmp_configs):
+        (tmp_configs / "models" / "empty.yaml").write_text("")
+        result = load_model_config("empty")
+        assert result == {}
+
+
+# ---------------------------------------------------------------------------
+# load_mode_config
+# ---------------------------------------------------------------------------
+
+
+class TestLoadModeConfig:
+    def test_loads_valid_yaml(self, tmp_configs):
+        data = {
+            "agent": {"agent_class": "eval.agents.gortex_agent.GortexAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        _write_yaml(tmp_configs / "modes" / "baseline.yaml", data)
+        result = load_mode_config("baseline")
+        assert result == data
+
+    def test_missing_config_raises(self, tmp_configs):
+        with pytest.raises(FileNotFoundError, match="Mode config not found"):
+            load_mode_config("nonexistent")
+
+
+# ---------------------------------------------------------------------------
+# merge_configs
+# ---------------------------------------------------------------------------
+
+
+class TestMergeConfigs:
+    def test_mode_overrides_model_on_conflict(self):
+        model = {"model": {"model_name": "old"}, "shared": "model_val"}
+        mode = {"shared": "mode_val"}
+        result = merge_configs(model, mode)
+        assert result["shared"] == "mode_val"
+        assert result["model"]["model_name"] == "old"
+
+    def test_deep_merge_nested_dicts(self):
+        model = {"agent": {"step_limit": 30, "cost_limit": 3.0}}
+        mode = {"agent": {"step_limit": 50, "extra": True}}
+        result = merge_configs(model, mode)
+        assert result["agent"]["step_limit"] == 50
+        assert result["agent"]["cost_limit"] == 3.0
+        assert result["agent"]["extra"] is True
+
+    def test_unique_keys_preserved(self):
+        model = {"model": {"model_name": "test"}}
+        mode = {"agent": {"agent_class": "MyAgent"}}
+        result = merge_configs(model, mode)
+        assert result["model"]["model_name"] == "test"
+        assert result["agent"]["agent_class"] == "MyAgent"
+
+    def test_empty_configs(self):
+        assert merge_configs({}, {}) == {}
+        assert merge_configs({"a": 1}, {}) == {"a": 1}
+        assert merge_configs({}, {"b": 2}) == {"b": 2}
+
+    def test_scalar_override_replaces_dict(self):
+        model = {"key": {"nested": "value"}}
+        mode = {"key": "scalar"}
+        result = merge_configs(model, mode)
+        assert result["key"] == "scalar"
+
+
+# ---------------------------------------------------------------------------
+# validate_config
+# ---------------------------------------------------------------------------
+
+
+class TestValidateConfig:
+    def test_valid_config_passes(self):
+        config = {
+            "model": {"model_name": "test"},
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        validate_config(config)  # should not raise
+
+    def test_missing_single_field(self):
+        config = {
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        with pytest.raises(ValueError, match="model.model_name"):
+            validate_config(config)
+
+    def test_missing_multiple_fields(self):
+        with pytest.raises(ValueError) as exc_info:
+            validate_config({})
+        msg = str(exc_info.value)
+        assert "model.model_name" in msg
+        assert "agent.agent_class" in msg
+        assert "environment.environment_class" in msg
+
+    def test_missing_nested_key(self):
+        config = {
+            "model": {},  # model_name missing inside model dict
+            "agent": {"agent_class": "MyAgent"},
+            "environment": {"environment_class": "docker"},
+        }
+        with pytest.raises(ValueError, match="model.model_name"):
+            validate_config(config)
+
+
+# ---------------------------------------------------------------------------
+# list_configs
+# ---------------------------------------------------------------------------
+
+
+class TestListConfigs:
+    def test_discovers_yaml_files(self, tmp_configs):
+        _write_yaml(tmp_configs / "models" / "claude-sonnet.yaml", {"model": {}})
+        _write_yaml(tmp_configs / "models" / "claude-haiku.yaml", {"model": {}})
+        _write_yaml(tmp_configs / "modes" / "baseline.yaml", {"agent": {}})
+        _write_yaml(tmp_configs / "modes" / "native.yaml", {"agent": {}})
+
+        result = list_configs()
+        assert "claude-haiku" in result["models"]
+        assert "claude-sonnet" in result["models"]
+        assert "baseline" in result["modes"]
+        assert "native" in result["modes"]
+
+    def test_excludes_non_yaml_files(self, tmp_configs):
+        _write_yaml(tmp_configs / "models" / "valid.yaml", {"model": {}})
+        (tmp_configs / "models" / ".gitkeep").write_text("")
+        (tmp_configs / "models" / "readme.txt").write_text("not yaml")
+
+        result = list_configs()
+        assert result["models"] == ["valid"]
+
+    def test_empty_directories(self, tmp_configs):
+        result = list_configs()
+        assert result == {"models": [], "modes": []}
+
+    def test_returns_sorted_names(self, tmp_configs):
+        for name in ["zebra", "alpha", "middle"]:
+            _write_yaml(tmp_configs / "models" / f"{name}.yaml", {})
+        result = list_configs()
+        assert result["models"] == ["alpha", "middle", "zebra"]

eval/tests/test_config_properties.py

@@ -0,0 +1,210 @@
+"""Property-based tests for eval.config module.
+
+Feature: eval-framework
+Uses hypothesis to verify config merge and validation properties.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from config import merge_configs, validate_config
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Keys: non-empty strings without dots (dots are used as path separators in
+# validate_config, so keeping keys simple avoids confusion).
+_key_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+    min_size=1,
+    max_size=12,
+)
+
+# Leaf values: scalars that YAML configs typically hold.
+_leaf_st = st.one_of(
+    st.text(min_size=0, max_size=30),
+    st.integers(min_value=-1000, max_value=1000),
+    st.floats(allow_nan=False, allow_infinity=False, min_value=-1e6, max_value=1e6),
+    st.booleans(),
+    st.none(),
+)
+
+# Shallow config dict (1 level deep) — sufficient for merge precedence tests.
+_flat_dict_st = st.dictionaries(keys=_key_st, values=_leaf_st, max_size=8)
+
+# Nested config dict (up to 2 levels) — mirrors real YAML configs.
+_nested_value_st = st.one_of(
+    _leaf_st,
+    st.dictionaries(keys=_key_st, values=_leaf_st, max_size=5),
+)
+_config_dict_st = st.dictionaries(keys=_key_st, values=_nested_value_st, max_size=8)
+
+
+# Required field paths used by validate_config.
+_REQUIRED_FIELDS = [
+    "model.model_name",
+    "agent.agent_class",
+    "environment.environment_class",
+]
+
+
+def _build_full_config() -> dict[str, Any]:
+    """Return a config dict that passes validation (all required fields present)."""
+    return {
+        "model": {"model_name": "test-model"},
+        "agent": {"agent_class": "eval.agents.TestAgent"},
+        "environment": {"environment_class": "eval.environments.TestEnv"},
+    }
+
+
+def _set_nested(d: dict, dotted_key: str, value: Any) -> None:
+    """Set a value in a nested dict using dot notation."""
+    parts = dotted_key.split(".")
+    current = d
+    for part in parts[:-1]:
+        current = current.setdefault(part, {})
+    current[parts[-1]] = value
+
+
+# ---------------------------------------------------------------------------
+# Property 10: Config merge precedence
+# Feature: eval-framework, Property 10: Config merge precedence
+# **Validates: Requirements 10.1, 10.2**
+# ---------------------------------------------------------------------------
+
+
+class TestConfigMergePrecedence:
+    """For any two config dicts, mode values override model values on shared
+    keys; unique keys preserved."""
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_mode_overrides_model_on_shared_keys(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """Shared top-level keys take the mode value (or deep-merged sub-dict)."""
+        merged = merge_configs(model, mode)
+
+        for key in mode:
+            if key in model:
+                model_val = model[key]
+                mode_val = mode[key]
+                merged_val = merged[key]
+
+                if isinstance(model_val, dict) and isinstance(mode_val, dict):
+                    # When both are dicts, mode sub-keys override model sub-keys.
+                    for sub_key in mode_val:
+                        assert merged_val[sub_key] == mode_val[sub_key]
+                else:
+                    # Scalar or type mismatch: mode wins entirely.
+                    assert merged_val == mode_val
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_unique_keys_preserved(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """Keys present in only one config appear unchanged in the merged result."""
+        merged = merge_configs(model, mode)
+
+        # Model-only keys preserved.
+        for key in model:
+            if key not in mode:
+                assert merged[key] == model[key]
+
+        # Mode-only keys preserved.
+        for key in mode:
+            if key not in model:
+                assert merged[key] == mode[key]
+
+    @given(model=_config_dict_st, mode=_config_dict_st)
+    @settings(max_examples=100)
+    def test_merged_contains_all_keys(
+        self, model: dict[str, Any], mode: dict[str, Any]
+    ) -> None:
+        """The merged dict contains every key from both inputs."""
+        merged = merge_configs(model, mode)
+        assert set(merged.keys()) == set(model.keys()) | set(mode.keys())
+
+
+# ---------------------------------------------------------------------------
+# Property 11: Config validation catches missing required fields
+# Feature: eval-framework, Property 11: Config validation catches missing required fields
+# **Validates: Requirements 10.3**
+# ---------------------------------------------------------------------------
+
+# Strategy: pick a non-empty subset of required fields to remove.
+_required_subsets_st = st.lists(
+    st.sampled_from(_REQUIRED_FIELDS),
+    min_size=1,
+    max_size=len(_REQUIRED_FIELDS),
+    unique=True,
+)
+
+
+class TestConfigValidationMissingFields:
+    """For any merged config missing one or more of (model_name, agent_class,
+    environment_class), validation fails naming the missing field(s)."""
+
+    @given(missing_fields=_required_subsets_st)
+    @settings(max_examples=100)
+    def test_validation_fails_naming_missing_fields(
+        self, missing_fields: list[str]
+    ) -> None:
+        """Removing any subset of required fields causes ValueError listing them."""
+        config = _build_full_config()
+
+        # Remove the selected required fields.
+        for field in missing_fields:
+            parts = field.split(".")
+            section = parts[0]
+            key = parts[1]
+            if section in config and isinstance(config[section], dict):
+                config[section].pop(key, None)
+
+        with pytest.raises(ValueError, match="Missing required config fields") as exc_info:
+            validate_config(config)
+
+        error_msg = str(exc_info.value)
+        for field in missing_fields:
+            assert field in error_msg, (
+                f"Expected '{field}' to be named in error but got: {error_msg}"
+            )
+
+    @given(missing_fields=_required_subsets_st)
+    @settings(max_examples=100)
+    def test_validation_only_reports_actually_missing_fields(
+        self, missing_fields: list[str]
+    ) -> None:
+        """Fields that ARE present should NOT appear in the error message."""
+        config = _build_full_config()
+        present_fields = [f for f in _REQUIRED_FIELDS if f not in missing_fields]
+
+        for field in missing_fields:
+            parts = field.split(".")
+            section = parts[0]
+            key = parts[1]
+            if section in config and isinstance(config[section], dict):
+                config[section].pop(key, None)
+
+        with pytest.raises(ValueError) as exc_info:
+            validate_config(config)
+
+        error_msg = str(exc_info.value)
+        for field in present_fields:
+            assert field not in error_msg, (
+                f"Field '{field}' is present but was reported as missing: {error_msg}"
+            )
+
+    @given(st.just(_build_full_config()))
+    @settings(max_examples=10)
+    def test_complete_config_passes_validation(self, config: dict[str, Any]) -> None:
+        """A config with all required fields should pass validation."""
+        validate_config(config)  # should not raise

eval/tests/test_docker_integration.py

@@ -0,0 +1,166 @@
+"""Docker container setup integration test.
+
+Feature: eval-framework
+Tests the GortexDockerEnvironment setup/teardown lifecycle with a real
+Docker daemon. Skipped when Docker is not available.
+
+This test validates:
+- Container launch with a lightweight image
+- Gortex binary copy into container
+- Eval-server health check
+- Container teardown and cleanup
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from environments.gortex_docker import GortexDockerEnvironment, _make_cache_key
+
+
+# --- Docker availability check ---
+
+def _docker_available() -> bool:
+    """Check if Docker daemon is accessible."""
+    try:
+        import docker
+        client = docker.from_env()
+        client.ping()
+        client.close()
+        return True
+    except Exception:
+        return False
+
+
+docker_available = _docker_available()
+
+
+# --- Integration tests (require Docker) ---
+
+@pytest.mark.skipif(
+    not docker_available,
+    reason="Docker daemon not available",
+)
+class TestDockerContainerLifecycle:
+    """Integration tests that exercise real Docker container lifecycle.
+
+    These tests require a running Docker daemon and will pull/use
+    lightweight images. They are skipped in CI environments without Docker.
+    """
+
+    def test_setup_teardown_gortex_disabled(self) -> None:
+        """Launch a container with gortex disabled, verify it runs, teardown."""
+        env = GortexDockerEnvironment(
+            image="alpine:latest",
+            enable_gortex=False,
+            instance_id="integration-test-no-gortex",
+        )
+        try:
+            result = env.setup()
+            # setup should succeed (no failure dict returned)
+            if result is not None:
+                pytest.skip(f"Container setup failed: {result.get('setup_error', 'unknown')}")
+
+            assert env._container is not None
+            assert env.is_ready
+
+            # Run a simple command inside the container
+            code, output = env.exec_run("echo hello-from-container")
+            assert code == 0
+            assert "hello-from-container" in output
+        finally:
+            env.teardown()
+            assert env._container is None
+
+    def test_extract_patch_empty_repo(self) -> None:
+        """Extract patch from a container with no git changes returns empty."""
+        env = GortexDockerEnvironment(
+            image="alpine:latest",
+            enable_gortex=False,
+            instance_id="integration-test-patch",
+        )
+        try:
+            result = env.setup()
+            if result is not None:
+                pytest.skip(f"Container setup failed: {result.get('setup_error', 'unknown')}")
+
+            # No git repo in alpine, so extract_patch should return empty
+            patch = env.extract_patch()
+            assert patch == ""
+        finally:
+            env.teardown()
+
+
+# --- Mock-based tests (always run, no Docker required) ---
+
+class TestDockerEnvironmentMocked:
+    """Tests that verify Docker integration logic using mocks.
+
+    These always run regardless of Docker availability.
+    """
+
+    @patch("environments.gortex_docker.docker")
+    def test_full_lifecycle_gortex_disabled(self, mock_docker) -> None:
+        """Verify setup → exec → extract_patch → teardown with mocked Docker."""
+        mock_client = MagicMock()
+        mock_container = MagicMock()
+        mock_container.short_id = "abc123"
+        mock_container.exec_run.return_value = (0, b"hello\n")
+        mock_client.containers.run.return_value = mock_container
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            enable_gortex=False,
+            instance_id="mock-lifecycle",
+        )
+
+        # Setup
+        result = env.setup()
+        assert result is None
+        assert env._container is mock_container
+
+        # Exec
+        code, output = env.exec_run("echo hello")
+        assert code == 0
+        assert output == "hello\n"
+
+        # Teardown
+        env.teardown()
+        mock_container.stop.assert_called_once()
+        mock_container.remove.assert_called_once()
+        assert env._container is None
+
+    @patch("environments.gortex_docker.docker")
+    def test_setup_failure_records_error(self, mock_docker) -> None:
+        """Verify that container launch failure is properly recorded."""
+        mock_client = MagicMock()
+        mock_client.containers.run.side_effect = RuntimeError("Docker daemon not running")
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            instance_id="fail-test",
+        )
+        result = env.setup()
+        assert result is not None
+        assert result["exit_status"] == "setup_failure"
+        assert "fail-test" in result["instance_id"]
+
+    def test_cache_key_determinism(self) -> None:
+        """Cache key for same inputs is always the same."""
+        k1 = _make_cache_key("repo", "abc123")
+        k2 = _make_cache_key("repo", "abc123")
+        assert k1 == k2
+
+    def test_cache_key_uniqueness(self) -> None:
+        """Different inputs produce different cache keys."""
+        k1 = _make_cache_key("repo_a", "commit1")
+        k2 = _make_cache_key("repo_b", "commit2")
+        assert k1 != k2

eval/tests/test_gortex_docker.py

@@ -0,0 +1,237 @@
+"""Unit tests for eval/environments/gortex_docker.py.
+
+Tests focus on pure logic (cache key, failure recording, properties)
+and mock Docker interactions to avoid requiring a running Docker daemon.
+"""
+
+from __future__ import annotations
+
+import io
+import tarfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from environments.gortex_docker import (
+    DEFAULT_EVAL_SERVER_PORT,
+    DEFAULT_GORTEX_TIMEOUT,
+    GortexDockerEnvironment,
+    _make_cache_key,
+)
+
+
+# -- _make_cache_key ---------------------------------------------------------
+
+class TestMakeCacheKey:
+    def test_basic(self):
+        assert _make_cache_key("django", "abc123") == "django_abc123"
+
+    def test_slash_in_repo_name(self):
+        assert _make_cache_key("django/django", "abc123") == "django__django_abc123"
+
+    def test_deterministic(self):
+        k1 = _make_cache_key("repo", "commit")
+        k2 = _make_cache_key("repo", "commit")
+        assert k1 == k2
+
+    def test_different_inputs_different_keys(self):
+        k1 = _make_cache_key("repo_a", "commit1")
+        k2 = _make_cache_key("repo_b", "commit2")
+        assert k1 != k2
+
+
+# -- GortexDockerEnvironment init -------------------------------------------
+
+class TestInit:
+    def test_defaults(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.image == "test:latest"
+        assert env.enable_gortex is True
+        assert env.gortex_timeout == DEFAULT_GORTEX_TIMEOUT
+        assert env.eval_server_port == DEFAULT_EVAL_SERVER_PORT
+        assert env._container is None
+        assert env._gortex_ready is False
+
+    def test_gortex_disabled(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        assert env.enable_gortex is False
+
+    def test_custom_params(self):
+        env = GortexDockerEnvironment(
+            image="swe:v1",
+            gortex_binary="/tmp/gortex",
+            gortex_timeout=60,
+            eval_server_port=9999,
+            cache_dir="/tmp/cache",
+            instance_id="django__django-1234",
+        )
+        assert env.gortex_binary == Path("/tmp/gortex")
+        assert env.gortex_timeout == 60
+        assert env.eval_server_port == 9999
+        assert env.cache_dir == Path("/tmp/cache")
+        assert env.instance_id == "django__django-1234"
+
+
+# -- setup / failure recording -----------------------------------------------
+
+class TestRecordFailure:
+    def test_returns_failure_dict(self):
+        env = GortexDockerEnvironment(image="test:latest", instance_id="test-123")
+        result = env._record_failure("something broke")
+        assert result["exit_status"] == "setup_failure"
+        assert result["instance_id"] == "test-123"
+        assert "something broke" in result["setup_error"]
+        assert result["submission"] == ""
+
+    def test_sets_setup_error(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        env._record_failure("timeout")
+        assert env.setup_error == "timeout"
+
+
+# -- is_ready property -------------------------------------------------------
+
+class TestIsReady:
+    def test_not_ready_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.is_ready is False
+
+    def test_ready_when_gortex_disabled(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        env._container = MagicMock()  # simulate running container
+        assert env.is_ready is True
+
+    def test_not_ready_gortex_enabled_but_not_setup(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=True)
+        env._container = MagicMock()
+        assert env.is_ready is False
+
+    def test_ready_gortex_enabled_and_setup(self):
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=True)
+        env._container = MagicMock()
+        env._gortex_ready = True
+        assert env.is_ready is True
+
+
+# -- extract_patch -----------------------------------------------------------
+
+class TestExtractPatch:
+    def test_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        assert env.extract_patch() == ""
+
+    def test_successful_diff(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"diff --git a/foo.py b/foo.py\n+hello\n")
+        env._container = mock_container
+        patch = env.extract_patch()
+        assert "diff --git" in patch
+        assert "+hello" in patch
+
+    def test_failed_diff(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (1, b"error")
+        env._container = mock_container
+        assert env.extract_patch() == ""
+
+    def test_exception_returns_empty(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.side_effect = RuntimeError("boom")
+        env._container = mock_container
+        assert env.extract_patch() == ""
+
+
+# -- teardown ----------------------------------------------------------------
+
+class TestTeardown:
+    def test_teardown_stops_and_removes(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_client = MagicMock()
+        env._container = mock_container
+        env._client = mock_client
+        env.teardown()
+        mock_container.stop.assert_called_once()
+        mock_container.remove.assert_called_once()
+        mock_client.close.assert_called_once()
+        assert env._container is None
+        assert env._client is None
+
+    def test_teardown_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        env.teardown()  # should not raise
+
+
+# -- exec_run ----------------------------------------------------------------
+
+class TestExecRun:
+    def test_no_container(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        code, output = env.exec_run("echo hello")
+        assert code == 1
+        assert "not running" in output.lower()
+
+    def test_string_command_wrapped_in_bash(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"hello\n")
+        env._container = mock_container
+        code, output = env.exec_run("echo hello")
+        assert code == 0
+        assert output == "hello\n"
+        # Verify it was wrapped in bash -c
+        call_args = mock_container.exec_run.call_args
+        assert call_args[0][0] == ["bash", "-c", "echo hello"]
+
+    def test_list_command_passed_directly(self):
+        env = GortexDockerEnvironment(image="test:latest")
+        mock_container = MagicMock()
+        mock_container.exec_run.return_value = (0, b"ok")
+        env._container = mock_container
+        code, output = env.exec_run(["ls", "-la"])
+        assert code == 0
+        call_args = mock_container.exec_run.call_args
+        assert call_args[0][0] == ["ls", "-la"]
+
+
+# -- setup with gortex disabled ---------------------------------------------
+
+class TestSetupGortexDisabled:
+    @patch("environments.gortex_docker.docker")
+    def test_setup_skips_gortex(self, mock_docker):
+        mock_client = MagicMock()
+        mock_container = MagicMock()
+        mock_container.short_id = "abc123"
+        mock_client.containers.run.return_value = mock_container
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(image="test:latest", enable_gortex=False)
+        result = env.setup()
+        assert result is None
+        assert env._container is mock_container
+
+
+# -- setup container launch failure ------------------------------------------
+
+class TestSetupContainerFailure:
+    @patch("environments.gortex_docker.docker")
+    def test_container_launch_failure(self, mock_docker):
+        mock_client = MagicMock()
+        mock_client.containers.run.side_effect = RuntimeError("Docker not running")
+        mock_docker.from_env.return_value = mock_client
+
+        env = GortexDockerEnvironment(
+            image="test:latest",
+            instance_id="fail-instance",
+        )
+        result = env.setup()
+        assert result is not None
+        assert result["exit_status"] == "setup_failure"
+        assert "fail-instance" in result["instance_id"]

eval/tests/test_prompts_properties.py

@@ -0,0 +1,86 @@
+"""Property-based tests for eval.prompts module.
+
+Feature: eval-framework
+Uses hypothesis to verify prompt template loading and rendering properties.
+"""
+
+from __future__ import annotations
+
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from prompts import VALID_MODES, load_templates, render_instance_prompt
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+_mode_st = st.sampled_from(VALID_MODES)
+
+# Non-empty text strings for task descriptions.  Keep them printable and
+# reasonably sized so rendered output stays manageable.
+_task_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z")),
+    min_size=1,
+    max_size=200,
+)
+
+
+# ---------------------------------------------------------------------------
+# Property 5: Template loading consistency
+# Feature: eval-framework, Property 5: Template loading consistency
+# **Validates: Requirements 2.4, 11.1**
+# ---------------------------------------------------------------------------
+
+
+class TestTemplateLoadingConsistency:
+    """For any valid mode name, the loader returns matching
+    ``system_{mode}.jinja`` and ``instance_{mode}.jinja``."""
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_load_returns_two_templates(self, mode: str) -> None:
+        """load_templates always returns a 2-tuple for every valid mode."""
+        result = load_templates(mode)
+        assert isinstance(result, tuple)
+        assert len(result) == 2
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_system_template_matches_mode(self, mode: str) -> None:
+        """The system template's filename matches ``system_{mode}.jinja``."""
+        system_tpl, _ = load_templates(mode)
+        assert system_tpl.name == f"system_{mode}.jinja"
+
+    @given(mode=_mode_st)
+    @settings(max_examples=100)
+    def test_instance_template_matches_mode(self, mode: str) -> None:
+        """The instance template's filename matches ``instance_{mode}.jinja``."""
+        _, instance_tpl = load_templates(mode)
+        assert instance_tpl.name == f"instance_{mode}.jinja"
+
+
+# ---------------------------------------------------------------------------
+# Property 12: Template rendering includes task
+# Feature: eval-framework, Property 12: Template rendering includes task
+# **Validates: Requirements 11.2**
+# ---------------------------------------------------------------------------
+
+
+class TestTemplateRenderingIncludesTask:
+    """For any non-empty task string, rendered instance prompt contains the
+    task verbatim."""
+
+    @given(mode=_mode_st, task=_task_st)
+    @settings(max_examples=100)
+    def test_rendered_output_contains_task_verbatim(
+        self, mode: str, task: str
+    ) -> None:
+        """The rendered instance prompt must contain the task string as-is."""
+        _, instance_tpl = load_templates(mode)
+        rendered = render_instance_prompt(instance_tpl, task)
+        assert task in rendered, (
+            f"Task string not found verbatim in rendered output.\n"
+            f"  task:     {task!r}\n"
+            f"  rendered: {rendered[:300]!r}..."
+        )

eval/tests/test_results_properties.py

@@ -0,0 +1,293 @@
+"""Property-based tests for eval.results module.
+
+Feature: eval-framework
+Uses hypothesis to verify result completeness, serialization, and aggregation.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from results import InstanceResult, RunSummary, save_run_summary
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+_MODES = ["baseline", "native", "native_augment"]
+
+_TOOL_NAMES = [
+    "search_symbols",
+    "smart_context",
+    "explain_change_impact",
+    "graph_stats",
+    "find_usages",
+]
+
+_EXIT_STATUSES = ["submitted", "setup_failure", "api_error", "cost_limit", "step_limit"]
+
+
+def _gortex_tool_calls_st() -> st.SearchStrategy[dict[str, int]]:
+    """Strategy for gortex_metrics.tool_calls dict."""
+    return st.fixed_dictionaries(
+        {name: st.integers(min_value=0, max_value=50) for name in _TOOL_NAMES}
+    )
+
+
+def _gortex_metrics_st(mode: str) -> st.SearchStrategy[dict[str, Any]]:
+    """Strategy for gortex_metrics based on mode."""
+    if mode == "baseline":
+        return st.just({})
+    elif mode == "native":
+        return st.builds(
+            lambda tc: {
+                "tool_calls": tc,
+                "total_tool_calls": sum(tc.values()),
+            },
+            tc=_gortex_tool_calls_st(),
+        )
+    else:  # native_augment
+        return st.builds(
+            lambda tc, aug_calls, aug_hits, aug_errors, aug_time, idx_time: {
+                "tool_calls": tc,
+                "total_tool_calls": sum(tc.values()),
+                "augmentation_calls": aug_calls,
+                "augmentation_hits": aug_hits,
+                "augmentation_errors": aug_errors,
+                "augmentation_time_seconds": aug_time,
+                "index_time_seconds": idx_time,
+            },
+            tc=_gortex_tool_calls_st(),
+            aug_calls=st.integers(min_value=0, max_value=100),
+            aug_hits=st.integers(min_value=0, max_value=100),
+            aug_errors=st.integers(min_value=0, max_value=20),
+            aug_time=st.floats(min_value=0.0, max_value=60.0, allow_nan=False, allow_infinity=False),
+            idx_time=st.floats(min_value=0.0, max_value=300.0, allow_nan=False, allow_infinity=False),
+        )
+
+
+@st.composite
+def instance_result_st(draw: st.DrawFn, mode: str | None = None) -> InstanceResult:
+    """Strategy that generates a valid InstanceResult for a given mode."""
+    m = mode if mode is not None else draw(st.sampled_from(_MODES))
+    return InstanceResult(
+        instance_id=draw(st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=30,
+        )),
+        model=draw(st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        )),
+        mode=m,
+        exit_status=draw(st.sampled_from(_EXIT_STATUSES)),
+        submission=draw(st.text(min_size=0, max_size=200)),
+        cost=draw(st.floats(min_value=0.0, max_value=100.0, allow_nan=False, allow_infinity=False)),
+        tokens_input=draw(st.integers(min_value=0, max_value=500_000)),
+        tokens_output=draw(st.integers(min_value=0, max_value=100_000)),
+        n_calls=draw(st.integers(min_value=0, max_value=200)),
+        n_steps=draw(st.integers(min_value=0, max_value=100)),
+        duration_seconds=draw(st.floats(min_value=0.0, max_value=3600.0, allow_nan=False, allow_infinity=False)),
+        gortex_metrics=draw(_gortex_metrics_st(m)),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Property 7: Result completeness per mode
+# Feature: eval-framework, Property 7: Result completeness per mode
+# **Validates: Requirements 6.1, 6.2, 6.3**
+# ---------------------------------------------------------------------------
+
+
+class TestResultCompletenessPerMode:
+    """Base fields always present; native/native_augment include gortex tool
+    metrics; native_augment includes augmentation metrics."""
+
+    _BASE_FIELDS = {
+        "instance_id", "model", "mode", "exit_status", "submission",
+        "cost", "tokens_input", "tokens_output", "n_calls", "n_steps",
+        "duration_seconds", "gortex_metrics",
+    }
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_base_fields_always_present(self, result: InstanceResult) -> None:
+        """All base metric fields are present regardless of mode."""
+        d = result.to_dict()
+        for field_name in self._BASE_FIELDS:
+            assert field_name in d, f"Missing base field: {field_name}"
+
+    @given(result=instance_result_st(mode="native"))
+    @settings(max_examples=100)
+    def test_native_mode_has_gortex_tool_metrics(self, result: InstanceResult) -> None:
+        """Native mode results include gortex tool call metrics."""
+        metrics = result.gortex_metrics
+        assert "tool_calls" in metrics, "native mode must have tool_calls"
+        assert "total_tool_calls" in metrics, "native mode must have total_tool_calls"
+        for tool_name in _TOOL_NAMES:
+            assert tool_name in metrics["tool_calls"], f"Missing tool: {tool_name}"
+
+    @given(result=instance_result_st(mode="native_augment"))
+    @settings(max_examples=100)
+    def test_native_augment_mode_has_augmentation_metrics(self, result: InstanceResult) -> None:
+        """native_augment mode results include both tool and augmentation metrics."""
+        metrics = result.gortex_metrics
+        # Tool metrics
+        assert "tool_calls" in metrics
+        assert "total_tool_calls" in metrics
+        # Augmentation metrics
+        assert "augmentation_calls" in metrics, "native_augment must have augmentation_calls"
+        assert "augmentation_hits" in metrics, "native_augment must have augmentation_hits"
+        assert "augmentation_errors" in metrics, "native_augment must have augmentation_errors"
+        assert "augmentation_time_seconds" in metrics, "native_augment must have augmentation_time_seconds"
+
+    @given(result=instance_result_st(mode="baseline"))
+    @settings(max_examples=100)
+    def test_baseline_mode_has_empty_gortex_metrics(self, result: InstanceResult) -> None:
+        """Baseline mode results have empty gortex_metrics."""
+        assert result.gortex_metrics == {}
+
+
+# ---------------------------------------------------------------------------
+# Property 8: Result serialization round-trip
+# Feature: eval-framework, Property 8: Result serialization round-trip
+# **Validates: Requirements 6.4**
+# ---------------------------------------------------------------------------
+
+
+class TestResultSerializationRoundTrip:
+    """Serialize → deserialize produces equivalent object with all fields preserved."""
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_instance_result_round_trip(self, result: InstanceResult) -> None:
+        """InstanceResult survives to_dict → from_dict round-trip."""
+        d = result.to_dict()
+        restored = InstanceResult.from_dict(d)
+        assert restored.to_dict() == d
+
+    @given(result=instance_result_st())
+    @settings(max_examples=100)
+    def test_instance_result_json_round_trip(self, result: InstanceResult) -> None:
+        """InstanceResult survives to_dict → JSON string → parse → from_dict."""
+        d = result.to_dict()
+        json_str = json.dumps(d)
+        parsed = json.loads(json_str)
+        restored = InstanceResult.from_dict(parsed)
+        assert restored.to_dict() == d
+
+    @given(
+        run_id=st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        ),
+        model=st.text(
+            alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+            min_size=1, max_size=20,
+        ),
+        mode=st.sampled_from(_MODES),
+    )
+    @settings(max_examples=100)
+    def test_run_summary_round_trip(self, run_id: str, model: str, mode: str) -> None:
+        """RunSummary survives to_dict → from_dict round-trip."""
+        summary = RunSummary(
+            run_id=run_id,
+            model=model,
+            mode=mode,
+            timestamp=1234567890.0,
+            config={"model": {"model_name": model}},
+            total_instances=10,
+            completed=8,
+            patch_rate=0.8,
+            total_cost=5.0,
+            mean_cost=0.5,
+            total_tokens=10000,
+            mean_tokens=1000.0,
+            total_duration_seconds=100.0,
+            mean_duration_seconds=10.0,
+        )
+        d = summary.to_dict()
+        restored = RunSummary.from_dict(d)
+        assert restored.to_dict() == d
+
+
+# ---------------------------------------------------------------------------
+# Property 9: Aggregate metric correctness
+# Feature: eval-framework, Property 9: Aggregate metric correctness
+# **Validates: Requirements 6.5, 7.1, 7.3**
+# ---------------------------------------------------------------------------
+
+
+class TestAggregateMetricCorrectness:
+    """patch_rate = patches/total, mean_cost = total_cost/count,
+    per-tool aggregations = sum of per-instance counts."""
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_patch_rate_equals_patches_over_total(self, results: list[InstanceResult]) -> None:
+        """patch_rate = count of results with non-empty submission / total."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        patches = sum(1 for r in results if r.submission)
+        expected_rate = patches / len(results)
+        assert abs(summary.patch_rate - expected_rate) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_cost_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_cost = total_cost / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_cost = sum(r.cost for r in results)
+        expected_mean = total_cost / len(results)
+        assert abs(summary.mean_cost - expected_mean) < 1e-9
+        assert abs(summary.total_cost - total_cost) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_tokens_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_tokens = total_tokens / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_tokens = sum(r.tokens_input + r.tokens_output for r in results)
+        expected_mean = total_tokens / len(results)
+        assert abs(summary.total_tokens - total_tokens) < 1e-9
+        assert abs(summary.mean_tokens - expected_mean) < 1e-9
+
+    @given(results=st.lists(instance_result_st(), min_size=1, max_size=20))
+    @settings(max_examples=100)
+    def test_mean_duration_equals_total_over_count(self, results: list[InstanceResult]) -> None:
+        """mean_duration = total_duration / instance count."""
+        for r in results:
+            r.model = "test-model"
+            r.mode = "native"
+
+        with tempfile.TemporaryDirectory() as td:
+            summary = save_run_summary(results, "test-run", {}, base_dir=Path(td))
+
+        total_duration = sum(r.duration_seconds for r in results)
+        expected_mean = total_duration / len(results)
+        assert abs(summary.total_duration_seconds - total_duration) < 1e-9
+        assert abs(summary.mean_duration_seconds - expected_mean) < 1e-9

eval/tests/test_runner_properties.py

@@ -0,0 +1,375 @@
+"""Property-based tests for eval runner (run_eval module).
+
+Feature: eval-framework
+Uses hypothesis to verify runner orchestration properties.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+from unittest.mock import patch, MagicMock
+
+import pytest
+from hypothesis import given, settings, assume
+from hypothesis import strategies as st
+
+from run_eval import parse_slice, build_matrix_configs, run_configuration
+
+
+# ---------------------------------------------------------------------------
+# Strategies
+# ---------------------------------------------------------------------------
+
+# Bounded integers suitable for slice components.
+_slice_int_st = st.integers(min_value=-50, max_value=50)
+
+# Optional slice int (None means omitted).
+_opt_slice_int_st = st.one_of(st.none(), _slice_int_st)
+
+# Simple identifier-like strings for model/mode names.
+_name_st = st.text(
+    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
+    min_size=1,
+    max_size=12,
+)
+
+# Non-empty lists of unique names.
+_name_list_st = st.lists(_name_st, min_size=1, max_size=6, unique=True)
+
+
+def _make_instance(instance_id: str) -> Dict[str, Any]:
+    """Create a minimal fake SWE-bench instance dict."""
+    return {
+        "instance_id": instance_id,
+        "problem_statement": f"Fix {instance_id}",
+    }
+
+
+
+# ---------------------------------------------------------------------------
+# Property 3: Slice parsing correctness
+# Feature: eval-framework, Property 3: Slice parsing correctness
+# **Validates: Requirements 1.5**
+# ---------------------------------------------------------------------------
+
+
+class TestSliceParsingCorrectness:
+    """For any valid slice spec, result matches Python's list[start:end] semantics."""
+
+    @given(start=_opt_slice_int_st, end=_opt_slice_int_st)
+    @settings(max_examples=100)
+    def test_two_part_slice_matches_python(
+        self, start: int | None, end: int | None
+    ) -> None:
+        """A 'start:end' spec produces the same sublist as list[start:end]."""
+        # Build the spec string.
+        start_str = "" if start is None else str(start)
+        end_str = "" if end is None else str(end)
+        spec = f"{start_str}:{end_str}"
+
+        # Reference list large enough to exercise the slice.
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[start:end]
+
+    @given(start=_opt_slice_int_st, end=_opt_slice_int_st, step=_slice_int_st)
+    @settings(max_examples=100)
+    def test_three_part_slice_matches_python(
+        self, start: int | None, end: int | None, step: int
+    ) -> None:
+        """A 'start:end:step' spec produces the same sublist as list[start:end:step]."""
+        assume(step != 0)  # step=0 is invalid for Python slices
+
+        start_str = "" if start is None else str(start)
+        end_str = "" if end is None else str(end)
+        spec = f"{start_str}:{end_str}:{step}"
+
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[start:end:step]
+
+    @given(end=st.integers(min_value=-50, max_value=50))
+    @settings(max_examples=100)
+    def test_single_value_treated_as_end(self, end: int) -> None:
+        """A single integer spec like '5' is treated as slice(None, 5)."""
+        spec = str(end)
+        ref = list(range(100))
+
+        parsed = parse_slice(spec)
+        assert ref[parsed] == ref[:end]
+
+    def test_empty_spec_selects_everything(self) -> None:
+        """An empty string selects all elements."""
+        ref = list(range(20))
+        parsed = parse_slice("")
+        assert ref[parsed] == ref
+
+
+
+# ---------------------------------------------------------------------------
+# Property 1: Instance execution completeness
+# Feature: eval-framework, Property 1: Instance execution completeness
+# **Validates: Requirements 1.1**
+# ---------------------------------------------------------------------------
+
+
+def _fake_process_instance(instance, config, output_dir, run_id, model_name, mode_name):
+    """A mock process_instance that returns a result dict without side effects."""
+    return {
+        "instance_id": instance["instance_id"],
+        "model": model_name,
+        "mode": mode_name,
+        "exit_status": "submitted",
+        "submission": "fake-patch",
+        "cost": 0.01,
+    }
+
+
+class TestInstanceExecutionCompleteness:
+    """For any N instances and worker count W >= 1, runner produces exactly N
+    result records."""
+
+    @given(
+        n=st.integers(min_value=1, max_value=20),
+        workers=st.integers(min_value=1, max_value=4),
+    )
+    @settings(max_examples=100)
+    def test_produces_exactly_n_results(self, n: int, workers: int) -> None:
+        """run_configuration returns exactly N results for N instances."""
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_fake_process_instance),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers
+            )
+
+        assert len(results) == n
+
+    @given(n=st.integers(min_value=1, max_value=15))
+    @settings(max_examples=100)
+    def test_each_instance_id_present(self, n: int) -> None:
+        """Every instance ID appears exactly once in the results."""
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_fake_process_instance),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), 1
+            )
+
+        result_ids = [r["instance_id"] for r in results]
+        expected_ids = [inst["instance_id"] for inst in instances]
+        assert sorted(result_ids) == sorted(expected_ids)
+
+
+
+# ---------------------------------------------------------------------------
+# Property 2: Matrix cross-product completeness
+# Feature: eval-framework, Property 2: Matrix cross-product completeness
+# **Validates: Requirements 1.3**
+# ---------------------------------------------------------------------------
+
+
+class TestMatrixCrossProductCompleteness:
+    """For M models and K modes, matrix produces exactly M x K unique
+    (model, mode) configs."""
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_produces_m_times_k_configs(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """build_matrix_configs returns exactly M * K pairs."""
+        configs = build_matrix_configs(models, modes)
+        assert len(configs) == len(models) * len(modes)
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_all_pairs_unique(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """Every (model, mode) pair in the result is unique."""
+        configs = build_matrix_configs(models, modes)
+        assert len(set(configs)) == len(configs)
+
+    @given(models=_name_list_st, modes=_name_list_st)
+    @settings(max_examples=100)
+    def test_every_model_mode_combination_present(
+        self, models: List[str], modes: List[str]
+    ) -> None:
+        """Every possible (model, mode) combination appears in the result."""
+        configs = build_matrix_configs(models, modes)
+        config_set = set(configs)
+        for model in models:
+            for mode in modes:
+                assert (model, mode) in config_set
+
+
+
+# ---------------------------------------------------------------------------
+# Property 4: Failure isolation
+# Feature: eval-framework, Property 4: Failure isolation
+# **Validates: Requirements 1.7**
+# ---------------------------------------------------------------------------
+
+
+class TestFailureIsolation:
+    """For N instances where K fail, runner still produces results for all
+    N-K non-failing instances plus K failure entries.
+
+    The parallel path (workers >= 2) in run_configuration catches exceptions
+    from process_instance and records them as error entries. We test with
+    workers=2 to exercise this failure isolation logic.
+    """
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_failure_isolation_produces_n_results(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Even when K instances fail, we get exactly N total result records."""
+        k = data.draw(st.integers(min_value=0, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        assert len(results) == n
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_non_failing_instances_have_results(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Non-failing instances produce normal result records."""
+        k = data.draw(st.integers(min_value=1, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        # Non-failing instances should have "submitted" status.
+        non_failing_results = [
+            r for r in results if r["instance_id"] not in failing_ids
+        ]
+        assert len(non_failing_results) == n - k
+        for r in non_failing_results:
+            assert r["exit_status"] == "submitted"
+
+    @given(
+        n=st.integers(min_value=2, max_value=15),
+        data=st.data(),
+    )
+    @settings(max_examples=100)
+    def test_failing_instances_recorded_as_errors(
+        self, n: int, data: st.DataObject
+    ) -> None:
+        """Failing instances are recorded with error status."""
+        k = data.draw(st.integers(min_value=1, max_value=n - 1))
+        instances = [_make_instance(f"test__test-{i}") for i in range(n)]
+        failing_ids = {inst["instance_id"] for inst in instances[:k]}
+
+        def _mock_process(instance, config, output_dir, run_id, model_name, mode_name):
+            iid = instance["instance_id"]
+            if iid in failing_ids:
+                raise RuntimeError(f"Simulated failure for {iid}")
+            return {
+                "instance_id": iid,
+                "model": model_name,
+                "mode": mode_name,
+                "exit_status": "submitted",
+                "submission": "patch",
+                "cost": 0.01,
+            }
+
+        with (
+            patch("run_eval._build_config", return_value={"agent": {}}),
+            patch("run_eval.generate_run_id", return_value="test_run_1"),
+            patch("run_eval.process_instance", side_effect=_mock_process),
+            patch("pathlib.Path.mkdir"),
+            patch("pathlib.Path.write_text"),
+        ):
+            from pathlib import Path
+
+            results = run_configuration(
+                "test-model", "baseline", instances, Path("/tmp/fake"), workers=2
+            )
+
+        # Failing instances should have "error" status.
+        error_results = [
+            r for r in results if r["instance_id"] in failing_ids
+        ]
+        assert len(error_results) == k
+        for r in error_results:
+            assert r["exit_status"] == "error"

eval/tests/test_smoke.py

@@ -0,0 +1,130 @@
+"""Smoke tests for the eval framework.
+
+Feature: eval-framework
+Verifies basic sanity of all major components:
+- Bridge scripts pass bash -n
+- All prompt templates render with sample data
+- list_configs discovers YAML files
+- gortex eval-server --help exits 0 (skipped if binary unavailable)
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from config import list_configs
+from prompts import VALID_MODES, load_templates, render_instance_prompt
+
+# --- Paths ---
+EVAL_DIR = Path(__file__).resolve().parent.parent
+BRIDGE_DIR = EVAL_DIR / "bridge"
+PROMPTS_DIR = EVAL_DIR / "prompts"
+
+
+# --- Bridge script bash -n tests ---
+
+BRIDGE_SCRIPTS = sorted(
+    p for p in BRIDGE_DIR.iterdir()
+    if p.is_file() and not p.name.endswith(".py") and not p.name.startswith("__")
+) if BRIDGE_DIR.is_dir() else []
+
+
+@pytest.mark.parametrize("script", BRIDGE_SCRIPTS, ids=lambda p: p.name)
+def test_bridge_script_bash_syntax(script: Path) -> None:
+    """Each bridge script must pass bash -n (syntax check)."""
+    result = subprocess.run(
+        ["bash", "-n", str(script)],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"bash -n failed for {script.name}: {result.stderr}"
+    )
+
+
+# --- Prompt template rendering tests ---
+
+SAMPLE_TASK = (
+    "Fix the bug in django/contrib/auth/models.py where "
+    "AbstractUser.clean() does not normalize the email address."
+)
+
+
+@pytest.mark.parametrize("mode", VALID_MODES)
+def test_prompt_templates_render_without_errors(mode: str) -> None:
+    """All prompt templates render without errors with sample data."""
+    system_tpl, instance_tpl = load_templates(mode)
+
+    # System template should render (may have no variables)
+    system_output = system_tpl.render()
+    assert len(system_output) > 0, f"system_{mode}.jinja rendered empty"
+
+    # Instance template should render with task variable
+    instance_output = render_instance_prompt(instance_tpl, SAMPLE_TASK)
+    assert len(instance_output) > 0, f"instance_{mode}.jinja rendered empty"
+    assert SAMPLE_TASK in instance_output, (
+        f"instance_{mode}.jinja did not include the task verbatim"
+    )
+
+
+# --- list_configs discovery tests ---
+
+def test_list_configs_discovers_yaml_files() -> None:
+    """list_configs discovers all YAML files in configs/."""
+    configs = list_configs()
+
+    assert "models" in configs
+    assert "modes" in configs
+
+    # We know at least these configs exist
+    assert len(configs["models"]) >= 2, (
+        f"Expected at least 2 model configs, found: {configs['models']}"
+    )
+    assert len(configs["modes"]) >= 3, (
+        f"Expected at least 3 mode configs, found: {configs['modes']}"
+    )
+
+    # Verify known configs are present
+    assert "claude-sonnet" in configs["models"]
+    assert "claude-haiku" in configs["models"]
+    assert "baseline" in configs["modes"]
+    assert "native" in configs["modes"]
+    assert "native_augment" in configs["modes"]
+
+
+# --- gortex eval-server --help test ---
+
+# Check if gortex binary is available
+_gortex_binary = shutil.which("gortex") or (
+    str(EVAL_DIR.parent / "gortex") if (EVAL_DIR.parent / "gortex").is_file() else None
+)
+
+
+@pytest.mark.skipif(
+    _gortex_binary is None,
+    reason="gortex binary not found in PATH or project root",
+)
+def test_gortex_eval_server_help_exits_zero() -> None:
+    """gortex eval-server --help exits 0."""
+    result = subprocess.run(
+        [_gortex_binary, "eval-server", "--help"],
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert result.returncode == 0, (
+        f"gortex eval-server --help failed:\n"
+        f"stdout: {result.stdout}\n"
+        f"stderr: {result.stderr}"
+    )
+    assert "eval-server" in result.stdout.lower() or "eval-server" in result.stderr.lower(), (
+        "Help output should mention eval-server"
+    )