""" Coverage tests for benchmarks/comparison/evaluator.py. Targets the 44 missing lines: - compare_configurations() with empty configurations - compare_configurations() with all-failed runs - compare_configurations() with successful runs - compare_configurations() with mixed success/fail - _evaluate_single_configuration() success and error paths - _calculate_average_metrics() with empty/single/multi results - _create_comparison_visualizations() with no successful results """ import pytest from unittest.mock import MagicMock, patch MODULE = "local_deep_research.benchmarks.comparison.evaluator" # --------------------------------------------------------------------------- # compare_configurations – guard clauses # --------------------------------------------------------------------------- class TestCompareConfigurationsEmpty: @patch(f"{MODULE}.os.makedirs") def test_empty_configurations_returns_error(self, mock_makedirs): from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations(query="test query", configurations=[]) assert "error" in result assert result["error"] == "No configurations provided" @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_single_config_all_failed( self, mock_write_json, mock_viz, mock_evaluate, mock_makedirs, tmp_path, ): mock_evaluate.return_value = { "success": False, "error": "LLM unreachable", } from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg1"}], output_dir=str(tmp_path), ) assert result["successful_configurations"] == 0 assert result["failed_configurations"] == 1 class TestCompareConfigurationsSuccess: @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_successful_run_included_in_results( self, mock_write_json, mock_viz, mock_combined_score, mock_avg_metrics, mock_evaluate, mock_makedirs, tmp_path, ): mock_evaluate.return_value = { "success": True, "quality_metrics": {"overall_quality": 0.8}, "speed_metrics": {"total_duration": 30.0}, "resource_metrics": {}, } mock_avg_metrics.return_value = { "quality_metrics": {"overall_quality": 0.8}, "speed_metrics": {"total_duration": 30.0}, "resource_metrics": {}, } mock_combined_score.return_value = 0.75 from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="research q", configurations=[{"name": "Cfg1"}], output_dir=str(tmp_path), ) assert result["successful_configurations"] == 1 assert result["failed_configurations"] == 0 assert len(result["results"]) == 1 assert result["results"][0]["overall_score"] == 0.75 @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_results_sorted_by_score_descending( self, mock_write_json, mock_viz, mock_combined_score, mock_avg_metrics, mock_evaluate, mock_makedirs, tmp_path, ): mock_evaluate.return_value = { "success": True, "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } mock_avg_metrics.return_value = { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } scores = [0.3, 0.9, 0.6] mock_combined_score.side_effect = scores from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) cfgs = [{"name": f"Cfg{i}"} for i in range(3)] result = compare_configurations( query="q", configurations=cfgs, output_dir=str(tmp_path) ) returned_scores = [r["overall_score"] for r in result["results"]] assert returned_scores == sorted(returned_scores, reverse=True) @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_repetitions_aggregated( self, mock_write_json, mock_viz, mock_combined_score, mock_avg_metrics, mock_evaluate, mock_makedirs, tmp_path, ): mock_evaluate.return_value = { "success": True, "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } mock_avg_metrics.return_value = { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } mock_combined_score.return_value = 0.5 from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg1"}], output_dir=str(tmp_path), repetitions=3, ) assert result["results"][0]["runs_completed"] == 3 # --------------------------------------------------------------------------- # compare_configurations – mixed success/failure # --------------------------------------------------------------------------- class TestCompareConfigurationsMixed: @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_mixed_run_counts_correct( self, mock_write_json, mock_viz, mock_combined_score, mock_avg_metrics, mock_evaluate, mock_makedirs, tmp_path, ): # First config succeeds, second fails mock_evaluate.side_effect = [ { "success": True, "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, }, {"success": False, "error": "fail"}, ] mock_avg_metrics.return_value = { "quality_metrics": {}, "speed_metrics": {}, "resource_metrics": {}, } mock_combined_score.return_value = 0.6 from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "Cfg1"}, {"name": "Cfg2"}], output_dir=str(tmp_path), ) assert result["successful_configurations"] == 1 assert result["failed_configurations"] == 1 @patch(f"{MODULE}.os.makedirs") @patch(f"{MODULE}._evaluate_single_configuration") @patch(f"{MODULE}._calculate_average_metrics") @patch(f"{MODULE}.calculate_combined_score") @patch(f"{MODULE}._create_comparison_visualizations") @patch( "local_deep_research.security.file_write_verifier.write_json_verified" ) def test_exception_during_run_marked_as_failed( self, mock_write_json, mock_viz, mock_combined_score, mock_avg_metrics, mock_evaluate, mock_makedirs, tmp_path, ): mock_evaluate.side_effect = RuntimeError("connection error") from local_deep_research.benchmarks.comparison.evaluator import ( compare_configurations, ) result = compare_configurations( query="q", configurations=[{"name": "CfgBad"}], output_dir=str(tmp_path), ) assert result["failed_configurations"] == 1 # --------------------------------------------------------------------------- # _calculate_average_metrics # --------------------------------------------------------------------------- class TestCalculateAverageMetrics: def test_empty_list_returns_empty_dict(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) result = _calculate_average_metrics([]) assert result == {} def test_single_result_averaged_correctly(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"overall_quality": 0.8}, "speed_metrics": {"total_duration": 30.0}, "resource_metrics": {"process_memory_max_mb": 256.0}, } ] avg = _calculate_average_metrics(results) assert avg["quality_metrics"]["overall_quality"] == 0.8 assert avg["speed_metrics"]["total_duration"] == 30.0 assert avg["resource_metrics"]["process_memory_max_mb"] == 256.0 def test_multiple_results_averaged(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"score": 0.6}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {"score": 0.8}, "speed_metrics": {}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) assert avg["quality_metrics"]["score"] == pytest.approx(0.7, abs=1e-9) def test_missing_keys_in_some_results_handled(self): from local_deep_research.benchmarks.comparison.evaluator import ( _calculate_average_metrics, ) results = [ { "quality_metrics": {"a": 0.5}, "speed_metrics": {}, "resource_metrics": {}, }, { "quality_metrics": {}, "speed_metrics": {"b": 1.0}, "resource_metrics": {}, }, ] avg = _calculate_average_metrics(results) # "a" only in first result → average of [0.5] = 0.5 assert avg["quality_metrics"]["a"] == 0.5 # "b" only in second result → average of [1.0] = 1.0 assert avg["speed_metrics"]["b"] == 1.0 # --------------------------------------------------------------------------- # _create_comparison_visualizations – no successful results # --------------------------------------------------------------------------- class TestCreateComparisonVisualizationsNoResults: @patch(f"{MODULE}.plt") def test_no_successful_results_returns_early(self, mock_plt, tmp_path): from local_deep_research.benchmarks.comparison.evaluator import ( _create_comparison_visualizations, ) report = { "results": [{"success": False, "error": "failed"}], } _create_comparison_visualizations( report, output_dir=str(tmp_path), timestamp="20240101_000000" ) mock_plt.figure.assert_not_called() # --------------------------------------------------------------------------- # _evaluate_single_configuration – error path # --------------------------------------------------------------------------- class TestEvaluateSingleConfigurationError: @patch(f"{MODULE}.SpeedProfiler") @patch(f"{MODULE}.ResourceMonitor") @patch(f"{MODULE}.get_llm") def test_llm_error_returns_failure_dict( self, mock_get_llm, mock_monitor, mock_profiler ): mock_get_llm.side_effect = RuntimeError("no LLM available") mock_profiler_instance = MagicMock() mock_profiler.return_value = mock_profiler_instance mock_monitor_instance = MagicMock() mock_monitor.return_value = mock_monitor_instance from local_deep_research.benchmarks.comparison.evaluator import ( _evaluate_single_configuration, ) result = _evaluate_single_configuration( query="test query", config={"name": "bad_config"}, ) assert result["success"] is False assert "no LLM available" in result["error"]