chore: import upstream snapshot with attribution

tests/benchmarks/test_calculation_coverage.py

@@ -0,0 +1,743 @@
+"""
+Coverage-focused tests for benchmarks/metrics/calculation.py.
+
+Targets the ~37% uncovered code, specifically:
+- evaluate_benchmark_quality (with mocked runner)
+- measure_execution_time (with mocked SearchSystem)
+- calculate_quality_metrics (delegates to evaluate_benchmark_quality)
+- calculate_speed_metrics (delegates to measure_execution_time)
+- calculate_resource_metrics edge cases not yet covered
+- calculate_combined_score with weights that need normalization
+- calculate_metrics: malformed JSON mixed with valid lines, integer confidence,
+  confidence as None, confidence as a list (TypeError path)
+"""
+
+import json
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# Helper to write JSONL files
+# ---------------------------------------------------------------------------
+
+
+def _write_jsonl(path, records):
+    """Write a list of dicts (or raw strings) to a JSONL file."""
+    with open(path, "w") as f:
+        for r in records:
+            if isinstance(r, str):
+                f.write(r + "\n")
+            else:
+                f.write(json.dumps(r) + "\n")
+
+
+# ===========================================================================
+# calculate_metrics – coverage-gap tests
+# ===========================================================================
+
+
+class TestCalculateMetricsCoverageGaps:
+    """Tests targeting lines not yet exercised by existing suites."""
+
+    def test_malformed_json_among_valid_lines_returns_error(self, tmp_path):
+        """A single malformed JSON line causes json.loads to raise,
+        which is caught by the broad except and returns an error dict."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "mixed.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                '{"is_correct": true}',
+                "NOT JSON AT ALL",
+            ],
+        )
+        result = calculate_metrics(str(results_file))
+        # The function catches *any* exception while iterating and returns error
+        assert "error" in result
+
+    def test_confidence_as_integer_value(self, tmp_path):
+        """Integer confidence (not string) is truthy when non-zero,
+        and int() on an int succeeds."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "int_conf.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"confidence": 75},
+                {"confidence": 25},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["average_confidence"] == 50
+
+    def test_confidence_none_is_skipped(self, tmp_path):
+        """confidence=None is falsy, so it should be skipped."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "none_conf.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"confidence": None},
+                {"confidence": "60"},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["average_confidence"] == 60
+
+    def test_confidence_as_list_triggers_type_error(self, tmp_path):
+        """confidence=[1,2] is truthy, but int([1,2]) raises TypeError,
+        which is caught and skipped."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "list_conf.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"confidence": [1, 2]},
+                {"confidence": "50"},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["average_confidence"] == 50
+
+    def test_confidence_as_float_string(self, tmp_path):
+        """confidence='85.5' – int('85.5') raises ValueError, skipped."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "float_conf.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"confidence": "85.5"},
+                {"confidence": "100"},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        # "85.5" is skipped (ValueError on int()), only "100" counted
+        assert metrics["average_confidence"] == 100
+
+    def test_no_categories_key_absent_from_metrics(self, tmp_path):
+        """When no result has 'category', metrics should not contain 'categories'."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "no_cat.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"is_correct": True},
+                {"is_correct": False},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert "categories" not in metrics
+
+    def test_processing_time_zero_included(self, tmp_path):
+        """processing_time=0 is in the result (key exists), so it counts."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "zero_time.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"processing_time": 0},
+                {"processing_time": 10.0},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["average_processing_time"] == 5.0
+
+    def test_multiple_categories(self, tmp_path):
+        """Three categories, each with different accuracy."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "multi_cat.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"is_correct": True, "category": "A"},
+                {"is_correct": True, "category": "A"},
+                {"is_correct": False, "category": "B"},
+                {"is_correct": True, "category": "C"},
+                {"is_correct": False, "category": "C"},
+                {"is_correct": False, "category": "C"},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["categories"]["A"]["accuracy"] == 1.0
+        assert metrics["categories"]["B"]["accuracy"] == 0.0
+        assert metrics["categories"]["C"]["accuracy"] == pytest.approx(1 / 3)
+
+    def test_large_number_of_results(self, tmp_path):
+        """Ensure it handles a larger dataset correctly."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "large.jsonl"
+        records = [
+            {"is_correct": i % 2 == 0, "processing_time": float(i)}
+            for i in range(100)
+        ]
+        _write_jsonl(results_file, records)
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["total_examples"] == 100
+        assert metrics["graded_examples"] == 100
+        assert metrics["correct"] == 50
+        assert metrics["accuracy"] == 0.5
+
+    def test_error_field_with_is_correct(self, tmp_path):
+        """A result can have both 'error' and 'is_correct'."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_metrics,
+        )
+
+        results_file = tmp_path / "both.jsonl"
+        _write_jsonl(
+            results_file,
+            [
+                {"is_correct": True, "error": "partial failure"},
+            ],
+        )
+        metrics = calculate_metrics(str(results_file))
+        assert metrics["error_count"] == 1
+        assert metrics["graded_examples"] == 1
+        assert metrics["correct"] == 1
+
+
+# ===========================================================================
+# evaluate_benchmark_quality – mocked runner
+# ===========================================================================
+
+
+class TestEvaluateBenchmarkQuality:
+    """Tests for evaluate_benchmark_quality with mocked run_simpleqa_benchmark."""
+
+    def test_returns_accuracy_and_quality_score(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        mock_results = {"metrics": {"accuracy": 0.75}}
+        with patch(
+            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+            return_value=mock_results,
+        ) as mock_run:
+            # Provide an output_dir so no tempdir is created
+            result = evaluate_benchmark_quality(
+                system_config={"iterations": 1},
+                num_examples=5,
+                output_dir=str(tmp_path),
+            )
+
+        assert result["accuracy"] == 0.75
+        assert result["quality_score"] == 0.75
+        mock_run.assert_called_once()
+
+    def test_uses_temp_dir_when_output_dir_none(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        mock_results = {"metrics": {"accuracy": 0.5}}
+        with patch(
+            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+            return_value=mock_results,
+        ):
+            result = evaluate_benchmark_quality(
+                system_config={},
+                num_examples=2,
+                output_dir=None,
+            )
+
+        assert result["accuracy"] == 0.5
+
+    def test_exception_returns_zero_scores(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+            side_effect=RuntimeError("boom"),
+        ):
+            result = evaluate_benchmark_quality(
+                system_config={},
+                num_examples=2,
+                output_dir=str(tmp_path),
+            )
+
+        assert result["accuracy"] == 0.0
+        assert result["quality_score"] == 0.0
+        assert "error" in result
+
+    def test_missing_metrics_key_defaults_to_zero(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        # run_simpleqa_benchmark returns dict without "metrics"
+        with patch(
+            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+            return_value={},
+        ):
+            result = evaluate_benchmark_quality(
+                system_config={},
+                num_examples=2,
+                output_dir=str(tmp_path),
+            )
+
+        assert result["accuracy"] == 0.0
+        assert result["quality_score"] == 0.0
+
+    def test_config_values_passed_to_runner(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+            return_value={"metrics": {"accuracy": 0.9}},
+        ) as mock_run:
+            evaluate_benchmark_quality(
+                system_config={
+                    "iterations": 5,
+                    "questions_per_iteration": 3,
+                    "search_strategy": "custom",
+                    "search_tool": "google",
+                    "model_name": "gpt-4",
+                    "provider": "openai",
+                },
+                num_examples=10,
+                output_dir=str(tmp_path),
+            )
+
+        call_kwargs = mock_run.call_args[1]
+        assert call_kwargs["num_examples"] == 10
+        search_config = call_kwargs["search_config"]
+        assert search_config["iterations"] == 5
+        assert search_config["questions_per_iteration"] == 3
+        assert search_config["search_strategy"] == "custom"
+        assert search_config["search_tool"] == "google"
+        assert search_config["model_name"] == "gpt-4"
+        assert search_config["provider"] == "openai"
+
+    def test_temp_dir_cleaned_up_on_success(self):
+        """When output_dir is None, a temp dir is created and cleaned up."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            evaluate_benchmark_quality,
+        )
+
+        created_dirs = []
+
+        original_mkdtemp = __import__("tempfile").mkdtemp
+
+        def tracking_mkdtemp(**kwargs):
+            d = original_mkdtemp(**kwargs)
+            created_dirs.append(d)
+            return d
+
+        with (
+            patch(
+                "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
+                return_value={"metrics": {"accuracy": 0.5}},
+            ),
+            patch(
+                "tempfile.mkdtemp",
+                side_effect=tracking_mkdtemp,
+            ),
+        ):
+            evaluate_benchmark_quality(
+                system_config={},
+                num_examples=1,
+                output_dir=None,
+            )
+
+        assert len(created_dirs) == 1
+        # The temp dir should have been cleaned up
+        assert not Path(created_dirs[0]).exists()
+
+
+# ===========================================================================
+# measure_execution_time – mocked SearchSystem
+# ===========================================================================
+
+
+class TestMeasureExecutionTime:
+    """Tests for measure_execution_time with mocked SearchSystem."""
+
+    def _patch_search_system(self, search_time=0.1):
+        """Return a context manager that patches AdvancedSearchSystem and its deps."""
+        from contextlib import ExitStack
+
+        mock_system = MagicMock()
+        mock_system.search.return_value = "result"
+
+        mock_cls = MagicMock(return_value=mock_system)
+
+        class CombinedPatcher:
+            def __enter__(self2):
+                self2.stack = ExitStack().__enter__()
+                self2.stack.enter_context(
+                    patch(
+                        "local_deep_research.config.llm_config.get_llm",
+                        return_value=MagicMock(),
+                    )
+                )
+                self2.stack.enter_context(
+                    patch(
+                        "local_deep_research.config.search_config.get_search",
+                        return_value=MagicMock(),
+                    )
+                )
+                self2.stack.enter_context(
+                    patch(
+                        "local_deep_research.search_system.AdvancedSearchSystem",
+                        mock_cls,
+                    )
+                )
+                return self2
+
+            def __exit__(self2, *args):
+                self2.stack.__exit__(*args)
+
+        return (
+            CombinedPatcher(),
+            mock_cls,
+            mock_system,
+        )
+
+    def test_basic_speed_measurement(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            measure_execution_time,
+        )
+
+        patcher, mock_cls, mock_system = self._patch_search_system()
+        with patcher:
+            result = measure_execution_time(
+                system_config={"iterations": 1},
+                query="test",
+                num_runs=1,
+            )
+
+        assert "average_time" in result
+        assert "speed_score" in result
+        assert "min_time" in result
+        assert "max_time" in result
+        assert result["average_time"] >= 0
+        assert 0 < result["speed_score"] <= 1.0
+        mock_system.search.assert_called_once_with("test", full_response=False)
+
+    def test_multiple_runs_averaged(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            measure_execution_time,
+        )
+
+        patcher, mock_cls, mock_system = self._patch_search_system()
+        with patcher:
+            result = measure_execution_time(
+                system_config={},
+                query="multi",
+                num_runs=3,
+            )
+
+        assert mock_system.search.call_count == 3
+        assert result["average_time"] >= 0
+        assert result["min_time"] <= result["max_time"]
+
+    def test_search_tool_override(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            measure_execution_time,
+        )
+
+        config = {"iterations": 2}
+        patcher, mock_cls, mock_system = self._patch_search_system()
+        with patcher:
+            measure_execution_time(
+                system_config=config,
+                search_tool="duckduckgo",
+                num_runs=1,
+            )
+
+        # search_tool should have been set on the config
+        assert config["search_tool"] == "duckduckgo"
+
+    def test_exception_returns_zero_scores(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            measure_execution_time,
+        )
+
+        mock_system = MagicMock()
+        mock_system.search.side_effect = RuntimeError("connection failed")
+        mock_cls = MagicMock(return_value=mock_system)
+
+        patcher, _, _ = self._patch_search_system()
+        # Override the mock_cls with our exception-raising one
+        with patcher:
+            with patch(
+                "local_deep_research.search_system.AdvancedSearchSystem",
+                mock_cls,
+            ):
+                result = measure_execution_time(
+                    system_config={},
+                    num_runs=1,
+                )
+
+        assert result["average_time"] == 0.0
+        assert result["speed_score"] == 0.0
+        assert "error" in result
+
+    def test_speed_score_formula(self):
+        """Verify speed_score = 1/(1 + avg_time/30)."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            measure_execution_time,
+        )
+
+        patcher, mock_cls, mock_system = self._patch_search_system()
+        mock_system.search.return_value = "r"
+
+        with patcher:
+            result = measure_execution_time(
+                system_config={},
+                num_runs=1,
+            )
+
+        avg = result["average_time"]
+        expected_score = 1.0 / (1.0 + avg / 30.0)
+        assert result["speed_score"] == pytest.approx(expected_score, abs=0.01)
+
+
+# ===========================================================================
+# calculate_quality_metrics – delegates to evaluate_benchmark_quality
+# ===========================================================================
+
+
+class TestCalculateQualityMetrics:
+    """Tests for calculate_quality_metrics."""
+
+    def test_returns_quality_and_accuracy(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_quality_metrics,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.metrics.calculation.evaluate_benchmark_quality",
+            return_value={"quality_score": 0.85, "accuracy": 0.85},
+        ):
+            result = calculate_quality_metrics(
+                system_config={"iterations": 1},
+                num_examples=5,
+                output_dir=str(tmp_path),
+            )
+
+        assert result["quality_score"] == 0.85
+        assert result["accuracy"] == 0.85
+
+    def test_defaults_to_zero_on_missing_keys(self, tmp_path):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_quality_metrics,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.metrics.calculation.evaluate_benchmark_quality",
+            return_value={},
+        ):
+            result = calculate_quality_metrics(
+                system_config={},
+                output_dir=str(tmp_path),
+            )
+
+        assert result["quality_score"] == 0.0
+        assert result["accuracy"] == 0.0
+
+
+# ===========================================================================
+# calculate_speed_metrics – delegates to measure_execution_time
+# ===========================================================================
+
+
+class TestCalculateSpeedMetrics:
+    """Tests for calculate_speed_metrics."""
+
+    def test_returns_speed_score_and_time(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_speed_metrics,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
+            return_value={"speed_score": 0.7, "average_time": 12.0},
+        ):
+            result = calculate_speed_metrics(
+                system_config={},
+                query="hello",
+                num_runs=2,
+            )
+
+        assert result["speed_score"] == 0.7
+        assert result["average_time"] == 12.0
+
+    def test_defaults_to_zero_on_missing_keys(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_speed_metrics,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
+            return_value={},
+        ):
+            result = calculate_speed_metrics(system_config={})
+
+        assert result["speed_score"] == 0.0
+        assert result["average_time"] == 0.0
+
+    def test_passes_search_tool_and_query(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_speed_metrics,
+        )
+
+        with patch(
+            "local_deep_research.benchmarks.metrics.calculation.measure_execution_time",
+            return_value={"speed_score": 0.5, "average_time": 20.0},
+        ) as mock_measure:
+            calculate_speed_metrics(
+                system_config={"iterations": 3},
+                query="deep query",
+                search_tool="brave",
+                num_runs=5,
+            )
+
+        mock_measure.assert_called_once_with(
+            system_config={"iterations": 3},
+            query="deep query",
+            search_tool="brave",
+            num_runs=5,
+        )
+
+
+# ===========================================================================
+# calculate_resource_metrics – additional coverage
+# ===========================================================================
+
+
+class TestCalculateResourceMetricsAdditional:
+    """Additional resource metric tests for coverage gaps."""
+
+    def test_search_tool_parameter_ignored_in_heuristic(self):
+        """search_tool and query params exist but don't affect the heuristic."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_resource_metrics,
+        )
+
+        config = {
+            "iterations": 2,
+            "questions_per_iteration": 2,
+            "max_results": 50,
+        }
+        r1 = calculate_resource_metrics(
+            config, query="query A", search_tool="brave"
+        )
+        r2 = calculate_resource_metrics(
+            config, query="query B", search_tool="google"
+        )
+        assert r1["resource_score"] == r2["resource_score"]
+        assert r1["estimated_complexity"] == r2["estimated_complexity"]
+
+    def test_fractional_max_results(self):
+        """max_results can be a float; formula still works."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_resource_metrics,
+        )
+
+        config = {
+            "iterations": 1,
+            "questions_per_iteration": 1,
+            "max_results": 25,
+        }
+        metrics = calculate_resource_metrics(config)
+        expected_complexity = 1 * 1 * (25 / 50)
+        assert metrics["estimated_complexity"] == pytest.approx(
+            expected_complexity
+        )
+
+
+# ===========================================================================
+# calculate_combined_score – additional coverage
+# ===========================================================================
+
+
+class TestCalculateCombinedScoreAdditional:
+    """Additional combined score tests for coverage gaps."""
+
+    def test_weights_with_only_some_matching_metrics(self):
+        """Weights for categories not in metrics are harmless."""
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_combined_score,
+        )
+
+        metrics = {"quality": {"quality_score": 1.0}}
+        weights = {"quality": 0.5, "speed": 0.3, "resource": 0.2}
+        score = calculate_combined_score(metrics, weights)
+        # Only quality matches: 1.0 * (0.5/1.0) = 0.5
+        assert score == pytest.approx(0.5)
+
+    def test_all_weights_equal(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_combined_score,
+        )
+
+        metrics = {
+            "quality": {"quality_score": 0.9},
+            "speed": {"speed_score": 0.6},
+            "resource": {"resource_score": 0.3},
+        }
+        weights = {"quality": 1, "speed": 1, "resource": 1}
+        score = calculate_combined_score(metrics, weights)
+        expected = (0.9 + 0.6 + 0.3) / 3.0
+        assert score == pytest.approx(expected)
+
+    def test_very_large_weights_still_normalize(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_combined_score,
+        )
+
+        metrics = {
+            "quality": {"quality_score": 0.8},
+            "speed": {"speed_score": 0.4},
+            "resource": {"resource_score": 0.2},
+        }
+        weights = {"quality": 1000, "speed": 500, "resource": 500}
+        score = calculate_combined_score(metrics, weights)
+        # norm: quality=0.5, speed=0.25, resource=0.25
+        expected = 0.8 * 0.5 + 0.4 * 0.25 + 0.2 * 0.25
+        assert score == pytest.approx(expected)
+
+    def test_empty_weights_dict_returns_zero(self):
+        from local_deep_research.benchmarks.metrics.calculation import (
+            calculate_combined_score,
+        )
+
+        metrics = {"quality": {"quality_score": 1.0}}
+        score = calculate_combined_score(metrics, weights={})
+        assert score == 0.0