""" Branch-coverage tests for benchmarks/comparison/evaluator.py. Targets the remaining 44 missing lines, focusing on branches not yet exercised by test_evaluator_coverage.py or test_comparison_evaluator.py: - compare_configurations: empty list early-return, all-runs-failed branch, exception-during-run path, repetitions averaging - _evaluate_single_configuration: full exception handler (error dict returned) - _calculate_average_metrics: multi-result averaging across all three metric categories, None-value filtering, disjoint metric keys - _create_comparison_visualizations: matplotlib calls with successful results - comparison with repetitions: multiple reps correctly produce avg_metrics """ import pytest from unittest.mock import MagicMock, patch MODULE = "local_deep_research.benchmarks.comparison.evaluator" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- _SUCCESSFUL_RUN = { "success": True, "quality_metrics": {"overall_quality": 0.8}, "speed_metrics": {"total_duration": 20.0}, "resource_metrics": {"memory_mb": 128.0}, } _FAILED_RUN = {"success": False, "error": "boom"} def _avg_metrics_from(run): return { "quality_metrics": run.get("quality_metrics", {}), "speed_metrics": run.get("speed_metrics", {}), "resource_metrics": run.get("resource_metrics", {}), } # --------------------------------------------------------------------------- # 1. test_compare_configurations_empty_list # --------------------------------------------------------------------------- class TestCompareConfigurationsEmptyList: """compare_configurations returns an error dict for an empty list.""" @patch(f"{MODULE}.os.makedirs") def test_empty_list_returns_error_dict(self, mock_makedirs): from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations(query="anything", configurations=[]) assert isinstance(result, dict) assert "error" in result assert result["error"] == "No configurations provided" @patch(f"{MODULE}.os.makedirs") def test_empty_list_does_not_call_evaluate(self, mock_makedirs): """_evaluate_single_configuration must not be called for empty input.""" from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) with patch(f"{MODULE}._evaluate_single_configuration") as mock_eval: compare_configurations(query="q", configurations=[]) mock_eval.assert_not_called() @patch(f"{MODULE}.os.makedirs") def test_empty_list_still_creates_output_dir(self, mock_makedirs): """makedirs is called even before the empty-list guard.""" from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) compare_configurations( query="q", configurations=[], output_dir="/tmp/out_branch_test" ) mock_makedirs.assert_called_once_with( "/tmp/out_branch_test", exist_ok=True ) # --------------------------------------------------------------------------- # 2. test_compare_configurations_all_runs_failed # --------------------------------------------------------------------------- class TestCompareConfigurationsAllRunsFailed: """When every repetition of a config fails the summary has success=False.""" @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_all_runs_failed_success_false( self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path ): from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "BadConfig"}], output_dir=str(tmp_path), repetitions=2, ) assert result["successful_configurations"] == 0 assert result["failed_configurations"] == 1 @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_all_runs_failed_result_summary_shape( self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path ): from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "BadConfig"}], output_dir=str(tmp_path), ) summary = result["results"][0] assert summary["success"] is False assert summary["runs_completed"] == 0 assert summary["runs_failed"] == 1 assert "error" in summary assert summary["error"] == "All runs failed" @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_exception_in_evaluate_treated_as_failed_run( self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path ): """An exception raised by _evaluate_single_configuration counts as a failure.""" mock_eval.side_effect = RuntimeError("network error") from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Crasher"}], output_dir=str(tmp_path), ) assert result["failed_configurations"] == 1 # The individual_results entry must contain the error string individual = result["results"][0]["individual_results"] assert len(individual) == 1 assert individual[0]["success"] is False assert "network error" in individual[0]["error"] # --------------------------------------------------------------------------- # 3. test_evaluate_single_configuration_exception # --------------------------------------------------------------------------- class TestEvaluateSingleConfigurationException: """_evaluate_single_configuration returns a proper error dict on exception.""" @patch(f"{MODULE}.SpeedProfiler") @patch(f"{MODULE}.ResourceMonitor") @patch(f"{MODULE}.get_llm") def test_exception_returns_success_false( self, mock_get_llm, mock_monitor_cls, mock_profiler_cls ): mock_get_llm.side_effect = ValueError("bad model name") mock_profiler = MagicMock() mock_profiler_cls.return_value = mock_profiler mock_monitor = MagicMock() mock_monitor_cls.return_value = mock_monitor from local_deep_research.benchmarks.comparison.evaluator import ( _evaluate_single_configuration, ) result = _evaluate_single_configuration( query="test query", config={"name": "cfg1"} ) assert result["success"] is False @patch(f"{MODULE}.SpeedProfiler") @patch(f"{MODULE}.ResourceMonitor") @patch(f"{MODULE}.get_llm") def test_exception_message_in_result( self, mock_get_llm, mock_monitor_cls, mock_profiler_cls ): mock_get_llm.side_effect = ValueError("bad model name") mock_profiler = MagicMock() mock_profiler_cls.return_value = mock_profiler mock_monitor = MagicMock() mock_monitor_cls.return_value = mock_monitor from local_deep_research.benchmarks.comparison.evaluator import ( _evaluate_single_configuration, ) result = _evaluate_single_configuration( query="test query", config={"name": "cfg1"} ) assert "error" in result assert "bad model name" in result["error"] @patch(f"{MODULE}.SpeedProfiler") @patch(f"{MODULE}.ResourceMonitor") @patch(f"{MODULE}.get_llm") def test_exception_result_contains_timing_and_resource_details( self, mock_get_llm, mock_monitor_cls, mock_profiler_cls ): """Even on error the result includes timing_details and resource_details.""" mock_get_llm.side_effect = Exception("oops") profiler_instance = MagicMock() profiler_instance.get_timings.return_value = {"llm_init": 0.5} mock_profiler_cls.return_value = profiler_instance monitor_instance = MagicMock() monitor_instance.get_combined_stats.return_value = {"cpu": 10.0} mock_monitor_cls.return_value = monitor_instance from local_deep_research.benchmarks.comparison.evaluator import ( _evaluate_single_configuration, ) result = _evaluate_single_configuration(query="q", config={}) assert "timing_details" in result assert "resource_details" in result @patch(f"{MODULE}.SpeedProfiler") @patch(f"{MODULE}.ResourceMonitor") @patch(f"{MODULE}.get_llm") def test_exception_stops_profiling( self, mock_get_llm, mock_monitor_cls, mock_profiler_cls ): """stop() must be called on both profiler and monitor even on error.""" mock_get_llm.side_effect = Exception("fail") profiler_instance = MagicMock() mock_profiler_cls.return_value = profiler_instance monitor_instance = MagicMock() mock_monitor_cls.return_value = monitor_instance from local_deep_research.benchmarks.comparison.evaluator import ( _evaluate_single_configuration, ) _evaluate_single_configuration(query="q", config={}) profiler_instance.stop.assert_called() monitor_instance.stop.assert_called() # --------------------------------------------------------------------------- # 4. test_calculate_average_metrics # --------------------------------------------------------------------------- class TestCalculateAverageMetrics: """_calculate_average_metrics averages correctly across all metric types.""" def test_quality_metrics_averaged(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"score": 0.4}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"score": 0.6}, "speed_metrics": {}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) assert avg["quality_metrics"]["score"] == pytest.approx(0.5) def test_speed_metrics_averaged(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {}, "speed_metrics": {"total_duration": 10.0}, "resource_metrics": {}, }, { "quality_metrics": {}, "speed_metrics": {"total_duration": 30.0}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) assert avg["speed_metrics"]["total_duration"] == pytest.approx(20.0) def test_resource_metrics_averaged(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {"memory_mb": 100.0}, }, { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {"memory_mb": 200.0}, }, ] avg = _calculate_average_metrics(results) assert avg["resource_metrics"]["memory_mb"] == pytest.approx(150.0) def test_none_values_filtered_out(self): """None values are skipped so the average uses only real numbers.""" from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"x": None}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"x": 0.8}, "speed_metrics": {}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) # Only the non-None value contributes assert avg["quality_metrics"]["x"] == pytest.approx(0.8) def test_disjoint_metric_keys_all_present(self): """Keys present in only one result still appear in the output.""" from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"key_a": 0.9}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"key_b": 0.3}, "speed_metrics": {}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) assert avg["quality_metrics"]["key_a"] == pytest.approx(0.9) assert avg["quality_metrics"]["key_b"] == pytest.approx(0.3) def test_empty_input_returns_empty_dict(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) assert _calculate_average_metrics([]) == {} def test_three_results_averaged_correctly(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"q": 0.2}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"q": 0.5}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"q": 0.8}, "speed_metrics": {}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) assert avg["quality_metrics"]["q"] == pytest.approx(0.5) def test_output_always_has_three_categories(self): """Result always contains quality_metrics, speed_metrics, resource_metrics keys.""" from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"x": 1.0}, "speed_metrics": {}, "resource_metrics": {}, } ] avg = _calculate_average_metrics(results) assert "quality_metrics" in avg assert "speed_metrics" in avg assert "resource_metrics" in avg # --------------------------------------------------------------------------- # 5. test_create_comparison_visualizations # --------------------------------------------------------------------------- class TestCreateComparisonVisualizations: """_create_comparison_visualizations makes the expected matplotlib calls.""" @patch(f"{MODULE}.plt") def test_no_successful_results_skips_all_plots(self, mock_plt, tmp_path): """Early return when there are no successful results.""" from local_deep_research.benchmarks.comparison.evaluator import ( _create_comparison_visualizations, ) report = {"results": [{"success": False, "error": "failed"}]} _create_comparison_visualizations( report, output_dir=str(tmp_path), timestamp="20240101_120000" ) mock_plt.figure.assert_not_called() mock_plt.barh.assert_not_called() mock_plt.savefig.assert_not_called() @patch(f"{MODULE}._create_spider_chart") @patch(f"{MODULE}._create_pareto_chart") @patch(f"{MODULE}._create_metric_comparison_chart") @patch(f"{MODULE}.plt") def test_successful_results_calls_savefig( self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path ): """With one successful result all sub-chart helpers are called.""" from local_deep_research.benchmarks.comparison.evaluator import ( _create_comparison_visualizations, ) mock_fig = MagicMock() mock_plt.figure.return_value = mock_fig report = { "results": [ { "success": True, "name": "Config A", "overall_score": 0.75, "avg_metrics": { "quality_metrics": {"overall_quality": 0.75}, "speed_metrics": {"total_duration": 15.0}, "resource_metrics": {}, }, } ] } _create_comparison_visualizations( report, output_dir=str(tmp_path), timestamp="20240101_000000" ) # Overall score bar chart triggers figure/barh/savefig mock_plt.figure.assert_called() mock_plt.savefig.assert_called() mock_plt.close.assert_called() # Sub-chart helpers must have been invoked once each mock_metric_chart.assert_called() mock_spider.assert_called_once() mock_pareto.assert_called_once() @patch(f"{MODULE}._create_spider_chart") @patch(f"{MODULE}._create_pareto_chart") @patch(f"{MODULE}._create_metric_comparison_chart") @patch(f"{MODULE}.plt") def test_config_names_extracted_correctly( self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path ): """Config names used in barh must come from result['name'].""" from local_deep_research.benchmarks.comparison.evaluator import ( _create_comparison_visualizations, ) mock_plt.figure.return_value = MagicMock() report = { "results": [ { "success": True, "name": "MySpecialConfig", "overall_score": 0.5, "avg_metrics": { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, }, } ] } _create_comparison_visualizations( report, output_dir=str(tmp_path), timestamp="ts1" ) # barh should have been called with the name in the y-axis list barh_call_args = mock_plt.barh.call_args assert barh_call_args is not None config_names_arg = barh_call_args[0][0] assert "MySpecialConfig" in config_names_arg # --------------------------------------------------------------------------- # 6. test_comparison_with_repetitions # --------------------------------------------------------------------------- class TestComparisonWithRepetitions: """compare_configurations with multiple repetitions averages metrics correctly.""" @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}.calculate_combined_score", return_value=0.7) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_three_repetitions_runs_completed( self, mock_write, mock_viz, mock_score, mock_eval, mock_makedirs, tmp_path, ): mock_eval.return_value = _SUCCESSFUL_RUN.copy() from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg"}], output_dir=str(tmp_path), repetitions=3, ) summary = result["results"][0] assert summary["runs_completed"] == 3 assert summary["runs_failed"] == 0 assert summary["success"] is True @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}.calculate_combined_score", return_value=0.7) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_partial_repetition_failure( self, mock_write, mock_viz, mock_score, mock_eval, mock_makedirs, tmp_path, ): """Only successful repetitions count toward runs_completed.""" mock_eval.side_effect = [ _SUCCESSFUL_RUN.copy(), _FAILED_RUN.copy(), _SUCCESSFUL_RUN.copy(), ] from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg"}], output_dir=str(tmp_path), repetitions=3, ) summary = result["results"][0] assert summary["runs_completed"] == 2 assert summary["runs_failed"] == 1 assert summary["success"] is True @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score", return_value=0.65) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_repetitions_pass_all_successful_runs_to_avg( self, mock_write, mock_viz, mock_score, mock_avg, mock_eval, mock_makedirs, tmp_path, ): """_calculate_average_metrics is called with the list of successful runs.""" run1 = _SUCCESSFUL_RUN.copy() run2 = _SUCCESSFUL_RUN.copy() mock_eval.side_effect = [run1, run2] mock_avg.return_value = { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) compare_configurations( query="q", configurations=[{"name": "Cfg"}], output_dir=str(tmp_path), repetitions=2, ) mock_avg.assert_called_once_with([run1, run2]) @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}.calculate_combined_score", return_value=0.8) @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_repetitions_field_in_report( self, mock_write, mock_viz, mock_score, mock_eval, mock_makedirs, tmp_path, ): """The top-level 'repetitions' key reflects the requested count.""" mock_eval.return_value = _SUCCESSFUL_RUN.copy() from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg"}], output_dir=str(tmp_path), repetitions=5, ) assert result["repetitions"] == 5 @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_two_configs_sorted_by_score( self, mock_write, mock_viz, mock_score, mock_eval, mock_makedirs, tmp_path, ): """Results are sorted descending by overall_score.""" mock_eval.return_value = _SUCCESSFUL_RUN.copy() mock_score.side_effect = [0.3, 0.9] from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Low"}, {"name": "High"}], output_dir=str(tmp_path), ) scores = [ r["overall_score"] for r in result["results"] if r.get("success") ] assert scores == sorted(scores, reverse=True)