learningcircuit--local-deep-research/tests/benchmarks/test_comparison_results.py

"""Tests for benchmarks/comparison/results.py.

Tests cover:
- Benchmark_results initialization
- add_result method
- get_all method
- get_best method with different sort options
- File I/O operations (_load_results, _save_results)
"""

import json
import pytest
from unittest.mock import patch, mock_open


class TestBenchmarkResultsInit:
    """Tests for Benchmark_results initialization."""

    def test_uses_default_filename(self):
        """Should use 'benchmark_results.json' when no file specified."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()
            assert br.results_file == "benchmark_results.json"

    def test_uses_custom_filename(self):
        """Should use provided filename."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results(results_file="custom_results.json")
            assert br.results_file == "custom_results.json"

    def test_loads_existing_results(self):
        """Should load results from existing file."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [{"model": "test-model", "accuracy_focused": 0.85}]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()
            assert br.results == existing_data
            assert len(br.results) == 1
            assert br.results[0]["model"] == "test-model"

    def test_returns_empty_list_when_file_missing(self):
        """Should return empty list when file doesn't exist."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()
            assert br.results == []


class TestAddResult:
    """Tests for add_result method."""

    def test_adds_result_with_all_fields(self):
        """Should create result dict with all required fields."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()

        with patch.object(br, "_save_results"):
            result = br.add_result(
                model="gpt-4",
                hardware="RTX 4090",
                accuracy_focused=0.92,
                accuracy_source=0.88,
                avg_time_per_question=15.5,
                context_window=128000,
                temperature=0.7,
                ldr_version="1.0.0",
                date_tested="2024-01-15",
                notes="Test run",
            )

            assert result is True
            assert len(br.results) == 1
            assert br.results[0]["model"] == "gpt-4"
            assert br.results[0]["hardware"] == "RTX 4090"
            assert br.results[0]["accuracy_focused"] == 0.92
            assert br.results[0]["accuracy_source"] == 0.88
            assert br.results[0]["avg_time_per_question"] == 15.5
            assert br.results[0]["context_window"] == 128000
            assert br.results[0]["temperature"] == 0.7
            assert br.results[0]["ldr_version"] == "1.0.0"
            assert br.results[0]["date_tested"] == "2024-01-15"
            assert br.results[0]["notes"] == "Test run"

    def test_appends_to_existing_results(self):
        """Should append to existing results list."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [{"model": "existing-model", "accuracy_focused": 0.80}]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()

        with patch.object(br, "_save_results"):
            br.add_result(
                model="new-model",
                hardware="RTX 3080",
                accuracy_focused=0.85,
                accuracy_source=0.82,
                avg_time_per_question=20.0,
                context_window=32000,
                temperature=0.5,
                ldr_version="1.0.0",
                date_tested="2024-01-16",
            )

            assert len(br.results) == 2
            assert br.results[0]["model"] == "existing-model"
            assert br.results[1]["model"] == "new-model"

    def test_calls_save_results_after_adding(self):
        """Should call _save_results after adding result."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()

        with patch.object(br, "_save_results") as mock_save:
            br.add_result(
                model="test-model",
                hardware="CPU",
                accuracy_focused=0.70,
                accuracy_source=0.65,
                avg_time_per_question=30.0,
                context_window=8000,
                temperature=0.3,
                ldr_version="1.0.0",
                date_tested="2024-01-17",
            )

            mock_save.assert_called_once()

    def test_returns_true_on_success(self):
        """Should return True after adding result."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()

        with patch.object(br, "_save_results"):
            result = br.add_result(
                model="test-model",
                hardware="CPU",
                accuracy_focused=0.70,
                accuracy_source=0.65,
                avg_time_per_question=30.0,
                context_window=8000,
                temperature=0.3,
                ldr_version="1.0.0",
                date_tested="2024-01-17",
            )

            assert result is True

    def test_default_notes_is_empty_string(self):
        """Should use empty string as default for notes."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()

        with patch.object(br, "_save_results"):
            br.add_result(
                model="test-model",
                hardware="CPU",
                accuracy_focused=0.70,
                accuracy_source=0.65,
                avg_time_per_question=30.0,
                context_window=8000,
                temperature=0.3,
                ldr_version="1.0.0",
                date_tested="2024-01-17",
                # notes not provided
            )

            assert br.results[0]["notes"] == ""


class TestGetAll:
    """Tests for get_all method."""

    def test_returns_empty_list_when_no_results(self):
        """Should return empty list when no results added."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()
            assert br.get_all() == []

    def test_returns_all_results(self):
        """Should return all added results."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [
            {"model": "model-1", "accuracy_focused": 0.80},
            {"model": "model-2", "accuracy_focused": 0.85},
            {"model": "model-3", "accuracy_focused": 0.90},
        ]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()
            all_results = br.get_all()

            assert len(all_results) == 3
            assert all_results[0]["model"] == "model-1"
            assert all_results[1]["model"] == "model-2"
            assert all_results[2]["model"] == "model-3"


class TestGetBest:
    """Tests for get_best method."""

    def test_sorts_by_accuracy_focused_descending(self):
        """Should sort by accuracy_focused in descending order by default."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [
            {
                "model": "model-low",
                "accuracy_focused": 0.70,
                "avg_time_per_question": 10,
            },
            {
                "model": "model-high",
                "accuracy_focused": 0.95,
                "avg_time_per_question": 20,
            },
            {
                "model": "model-mid",
                "accuracy_focused": 0.85,
                "avg_time_per_question": 15,
            },
        ]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()
            best = br.get_best()

            assert len(best) == 3
            assert best[0]["model"] == "model-high"  # 0.95 first
            assert best[1]["model"] == "model-mid"  # 0.85 second
            assert best[2]["model"] == "model-low"  # 0.70 last

    def test_sorts_by_time_ascending(self):
        """Should sort avg_time_per_question in ascending order."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [
            {
                "model": "model-slow",
                "accuracy_focused": 0.90,
                "avg_time_per_question": 30.0,
            },
            {
                "model": "model-fast",
                "accuracy_focused": 0.85,
                "avg_time_per_question": 5.0,
            },
            {
                "model": "model-mid",
                "accuracy_focused": 0.88,
                "avg_time_per_question": 15.0,
            },
        ]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()
            best = br.get_best(sort_by="avg_time_per_question")

            assert len(best) == 3
            assert best[0]["model"] == "model-fast"  # 5.0 first (fastest)
            assert best[1]["model"] == "model-mid"  # 15.0 second
            assert best[2]["model"] == "model-slow"  # 30.0 last (slowest)

    def test_raises_value_error_for_invalid_key(self):
        """Should raise ValueError for invalid sort_by key."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [
            {
                "model": "test",
                "accuracy_focused": 0.85,
                "avg_time_per_question": 10,
            },
        ]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()

            with pytest.raises(ValueError) as excinfo:
                br.get_best(sort_by="invalid_key")

            assert "Invalid sort_by key: 'invalid_key'" in str(excinfo.value)
            assert "Valid keys are:" in str(excinfo.value)

    def test_handles_empty_results(self):
        """Should handle empty results list gracefully."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()
            best = br.get_best()

            assert best == []

    def test_sorts_by_other_numeric_fields_descending(self):
        """Should sort other fields in descending order (not just accuracy_focused)."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        existing_data = [
            {
                "model": "model-small",
                "accuracy_focused": 0.85,
                "context_window": 8000,
            },
            {
                "model": "model-large",
                "accuracy_focused": 0.90,
                "context_window": 128000,
            },
            {
                "model": "model-mid",
                "accuracy_focused": 0.88,
                "context_window": 32000,
            },
        ]
        mock_file = mock_open(read_data=json.dumps(existing_data))

        with patch("builtins.open", mock_file):
            br = Benchmark_results()
            best = br.get_best(sort_by="context_window")

            assert best[0]["model"] == "model-large"  # 128000 first
            assert best[1]["model"] == "model-mid"  # 32000 second
            assert best[2]["model"] == "model-small"  # 8000 last


class TestSaveResults:
    """Tests for _save_results method."""

    def test_calls_write_json_verified(self):
        """Should use write_json_verified for secure writes."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results(results_file="test_results.json")

        # Patch where write_json_verified is imported from (inside _save_results)
        with patch(
            "local_deep_research.security.file_write_verifier.write_json_verified"
        ) as mock_write:
            br._save_results()

            mock_write.assert_called_once_with(
                "test_results.json",
                [],
                "benchmark.allow_file_output",
                context="benchmark results",
            )

    def test_saves_current_results(self):
        """Should save current results list to file."""
        from local_deep_research.benchmarks.comparison.results import (
            Benchmark_results,
        )

        with patch("builtins.open", side_effect=FileNotFoundError):
            br = Benchmark_results()
            br.results = [{"model": "test", "accuracy_focused": 0.90}]

        # Patch where write_json_verified is imported from (inside _save_results)
        with patch(
            "local_deep_research.security.file_write_verifier.write_json_verified"
        ) as mock_write:
            br._save_results()

            # Check the results passed to write_json_verified
            call_args = mock_write.call_args
            assert call_args[0][1] == [
                {"model": "test", "accuracy_focused": 0.90}
            ]