learningcircuit--local-deep-research/tests/benchmarks/evaluators/test_composite.py

"""
Tests for CompositeBenchmarkEvaluator class.

Tests the composite benchmark evaluator that combines multiple benchmarks.
"""

import tempfile
from unittest.mock import MagicMock, patch


from local_deep_research.benchmarks.evaluators.composite import (
    CompositeBenchmarkEvaluator,
)


class TestCompositeBenchmarkEvaluatorInit:
    """Test initialization of CompositeBenchmarkEvaluator."""

    def test_init_default_weights(self):
        """Test initialization with default weights."""
        evaluator = CompositeBenchmarkEvaluator()
        assert "simpleqa" in evaluator.benchmark_weights
        assert evaluator.benchmark_weights["simpleqa"] == 1.0

    def test_init_custom_weights(self):
        """Test initialization with custom weights."""
        weights = {"simpleqa": 0.6, "browsecomp": 0.4}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
        assert evaluator.benchmark_weights == weights

    def test_init_normalizes_weights(self):
        """Test that initialization normalizes weights to sum to 1.0."""
        weights = {"simpleqa": 2.0, "browsecomp": 2.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        # Normalized weights should be 0.5 each
        assert evaluator.normalized_weights["simpleqa"] == 0.5
        assert evaluator.normalized_weights["browsecomp"] == 0.5

    def test_init_creates_evaluators(self):
        """Test that initialization creates evaluator instances."""
        evaluator = CompositeBenchmarkEvaluator()
        assert "simpleqa" in evaluator.evaluators
        assert "browsecomp" in evaluator.evaluators

    def test_init_handles_zero_total_weight(self):
        """Test initialization handles zero total weight."""
        weights = {"simpleqa": 0.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        # Should fall back to default weights
        assert evaluator.normalized_weights == {"simpleqa": 1.0}

    def test_init_handles_negative_weight(self):
        """Test initialization handles negative total weight."""
        weights = {"simpleqa": -1.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        # Should fall back to default weights
        assert evaluator.normalized_weights == {"simpleqa": 1.0}


class TestCompositeBenchmarkEvaluatorWeightNormalization:
    """Test weight normalization."""

    def test_normalize_single_weight(self):
        """Test normalization with single weight."""
        weights = {"simpleqa": 5.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
        assert evaluator.normalized_weights["simpleqa"] == 1.0

    def test_normalize_multiple_weights(self):
        """Test normalization with multiple weights."""
        weights = {"simpleqa": 3.0, "browsecomp": 1.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        assert evaluator.normalized_weights["simpleqa"] == 0.75
        assert evaluator.normalized_weights["browsecomp"] == 0.25

    def test_normalize_equal_weights(self):
        """Test normalization with equal weights."""
        weights = {"simpleqa": 1.0, "browsecomp": 1.0}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        assert evaluator.normalized_weights["simpleqa"] == 0.5
        assert evaluator.normalized_weights["browsecomp"] == 0.5

    def test_normalize_unequal_weights(self):
        """Test normalization with unequal weights."""
        weights = {"simpleqa": 0.7, "browsecomp": 0.3}
        evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)

        assert evaluator.normalized_weights["simpleqa"] == 0.7
        assert evaluator.normalized_weights["browsecomp"] == 0.3


class TestCompositeBenchmarkEvaluate:
    """Test evaluate method."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_runs_benchmarks_with_weight(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that evaluate runs benchmarks with positive weight."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.8}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {"quality_score": 0.6}
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.5, "browsecomp": 0.5}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            evaluator.evaluate(
                system_config={"key": "value"},
                num_examples=10,
                output_dir=tmpdir,
            )

            mock_simpleqa.evaluate.assert_called_once()
            mock_browsecomp.evaluate.assert_called_once()

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_computes_weighted_score(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that evaluate computes weighted combined score."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.8}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {"quality_score": 0.4}
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.6, "browsecomp": 0.4}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            # Expected: 0.8 * 0.6 + 0.4 * 0.4 = 0.48 + 0.16 = 0.64
            assert abs(result["quality_score"] - 0.64) < 0.001
            assert abs(result["combined_score"] - 0.64) < 0.001

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_returns_individual_results(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that evaluate returns individual benchmark results."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {
            "quality_score": 0.9,
            "accuracy": 0.9,
        }
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {
            "quality_score": 0.7,
            "accuracy": 0.7,
        }
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.5, "browsecomp": 0.5}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert "benchmark_results" in result
            assert "simpleqa" in result["benchmark_results"]
            assert "browsecomp" in result["benchmark_results"]

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_returns_weights_used(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that evaluate returns the weights used."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.5}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {"quality_score": 0.5}
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.7, "browsecomp": 0.3}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            assert "benchmark_weights" in result
            assert result["benchmark_weights"]["simpleqa"] == 0.7
            assert result["benchmark_weights"]["browsecomp"] == 0.3


class TestCompositeBenchmarkEvaluateErrors:
    """Test error handling in evaluate method."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_handles_evaluator_exception(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that evaluate handles exceptions from individual evaluators."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.side_effect = RuntimeError("SimpleQA failed")
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {"quality_score": 0.6}
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.5, "browsecomp": 0.5}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            # Should still have results
            assert "benchmark_results" in result
            assert "error" in result["benchmark_results"]["simpleqa"]
            assert (
                result["benchmark_results"]["browsecomp"]["quality_score"]
                == 0.6
            )

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_zero_score_on_error(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that failed benchmark contributes zero to combined score."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.side_effect = RuntimeError("Failed")
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp.evaluate.return_value = {"quality_score": 1.0}
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 0.5, "browsecomp": 0.5}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            # Only browsecomp contributes: 1.0 * 0.5 = 0.5
            assert result["quality_score"] == 0.5


class TestCompositeBenchmarkSingleEvaluator:
    """Test composite with single evaluator."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_evaluate_single_benchmark(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that composite can run with single benchmark."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.75}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        # Only simpleqa
        weights = {"simpleqa": 1.0}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            mock_simpleqa.evaluate.assert_called_once()
            mock_browsecomp.evaluate.assert_not_called()
            assert result["quality_score"] == 0.75


class TestCompositeBenchmarkMissingEvaluator:
    """Test handling of unknown benchmark names."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_unknown_benchmark_ignored(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that unknown benchmark names are ignored."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.8}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        # Include unknown benchmark
        weights = {"simpleqa": 0.5, "unknown_benchmark": 0.5}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            result = evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            # Only simpleqa should run
            mock_simpleqa.evaluate.assert_called_once()
            # Score should reflect only simpleqa's contribution
            assert "benchmark_results" in result


class TestCompositeBenchmarkZeroWeight:
    """Test handling of zero weights."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_zero_weight_benchmark_not_run(
        self, mock_browsecomp_cls, mock_simpleqa_cls
    ):
        """Test that benchmark with zero weight is not run."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.8}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        # browsecomp has zero weight
        weights = {"simpleqa": 1.0, "browsecomp": 0.0}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            mock_simpleqa.evaluate.assert_called_once()
            mock_browsecomp.evaluate.assert_not_called()


class TestCompositeBenchmarkPassesConfig:
    """Test that configuration is passed to evaluators."""

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_passes_system_config(self, mock_browsecomp_cls, mock_simpleqa_cls):
        """Test that system_config is passed to evaluators."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.5}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        config = {"iterations": 5, "search_tool": "google"}
        weights = {"simpleqa": 1.0}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            evaluator.evaluate(
                system_config=config,
                num_examples=10,
                output_dir=tmpdir,
            )

            call_kwargs = mock_simpleqa.evaluate.call_args[1]
            assert call_kwargs["system_config"] == config

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_passes_num_examples(self, mock_browsecomp_cls, mock_simpleqa_cls):
        """Test that num_examples is passed to evaluators."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.5}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 1.0}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            evaluator.evaluate(
                system_config={},
                num_examples=25,
                output_dir=tmpdir,
            )

            call_kwargs = mock_simpleqa.evaluate.call_args[1]
            assert call_kwargs["num_examples"] == 25

    @patch(
        "local_deep_research.benchmarks.evaluators.composite.SimpleQAEvaluator"
    )
    @patch(
        "local_deep_research.benchmarks.evaluators.composite.BrowseCompEvaluator"
    )
    def test_passes_output_dir(self, mock_browsecomp_cls, mock_simpleqa_cls):
        """Test that output_dir is passed to evaluators."""
        mock_simpleqa = MagicMock()
        mock_simpleqa.evaluate.return_value = {"quality_score": 0.5}
        mock_simpleqa_cls.return_value = mock_simpleqa

        mock_browsecomp = MagicMock()
        mock_browsecomp_cls.return_value = mock_browsecomp

        weights = {"simpleqa": 1.0}

        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = CompositeBenchmarkEvaluator(benchmark_weights=weights)
            evaluator.evaluate(
                system_config={},
                num_examples=5,
                output_dir=tmpdir,
            )

            call_kwargs = mock_simpleqa.evaluate.call_args[1]
            assert call_kwargs["output_dir"] == tmpdir