wehub-resource-sync
7a0da7932b
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Has been cancelled
Docker Tests (Consolidated) / Accessibility Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Unit Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Example Tests (push) Has been cancelled
Docker Tests (Consolidated) / Production Image Smoke Test (push) Has been cancelled
Docker Tests (Consolidated) / Infrastructure Tests (push) Has been cancelled
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Has been cancelled
Backwards Compatibility / Verify Encryption Constants (push) Has been cancelled
Backwards Compatibility / PyPI Version Compatibility (push) Has been cancelled
Backwards Compatibility / Database Migration Tests (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Docker Tests (Consolidated) / detect-changes (push) Has been cancelled
Docker Tests (Consolidated) / Build Test Image (push) Has been cancelled
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Has been cancelled
738 lines
25 KiB
Python
738 lines
25 KiB
Python
"""
|
|
Branch-coverage tests for benchmarks/comparison/evaluator.py.
|
|
|
|
Targets the remaining 44 missing lines, focusing on branches not yet
|
|
exercised by test_evaluator_coverage.py or test_comparison_evaluator.py:
|
|
|
|
- compare_configurations: empty list early-return, all-runs-failed branch,
|
|
exception-during-run path, repetitions averaging
|
|
- _evaluate_single_configuration: full exception handler (error dict returned)
|
|
- _calculate_average_metrics: multi-result averaging across all three metric
|
|
categories, None-value filtering, disjoint metric keys
|
|
- _create_comparison_visualizations: matplotlib calls with successful results
|
|
- comparison with repetitions: multiple reps correctly produce avg_metrics
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
MODULE = "local_deep_research.benchmarks.comparison.evaluator"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SUCCESSFUL_RUN = {
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 20.0},
|
|
"resource_metrics": {"memory_mb": 128.0},
|
|
}
|
|
|
|
_FAILED_RUN = {"success": False, "error": "boom"}
|
|
|
|
|
|
def _avg_metrics_from(run):
|
|
return {
|
|
"quality_metrics": run.get("quality_metrics", {}),
|
|
"speed_metrics": run.get("speed_metrics", {}),
|
|
"resource_metrics": run.get("resource_metrics", {}),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. test_compare_configurations_empty_list
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompareConfigurationsEmptyList:
|
|
"""compare_configurations returns an error dict for an empty list."""
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
def test_empty_list_returns_error_dict(self, mock_makedirs):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(query="anything", configurations=[])
|
|
|
|
assert isinstance(result, dict)
|
|
assert "error" in result
|
|
assert result["error"] == "No configurations provided"
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
def test_empty_list_does_not_call_evaluate(self, mock_makedirs):
|
|
"""_evaluate_single_configuration must not be called for empty input."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
with patch(f"{MODULE}._evaluate_single_configuration") as mock_eval:
|
|
compare_configurations(query="q", configurations=[])
|
|
mock_eval.assert_not_called()
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
def test_empty_list_still_creates_output_dir(self, mock_makedirs):
|
|
"""makedirs is called even before the empty-list guard."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
compare_configurations(
|
|
query="q", configurations=[], output_dir="/tmp/out_branch_test"
|
|
)
|
|
mock_makedirs.assert_called_once_with(
|
|
"/tmp/out_branch_test", exist_ok=True
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. test_compare_configurations_all_runs_failed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompareConfigurationsAllRunsFailed:
|
|
"""When every repetition of a config fails the summary has success=False."""
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_all_runs_failed_success_false(
|
|
self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
|
|
):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "BadConfig"}],
|
|
output_dir=str(tmp_path),
|
|
repetitions=2,
|
|
)
|
|
|
|
assert result["successful_configurations"] == 0
|
|
assert result["failed_configurations"] == 1
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_all_runs_failed_result_summary_shape(
|
|
self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
|
|
):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "BadConfig"}],
|
|
output_dir=str(tmp_path),
|
|
)
|
|
|
|
summary = result["results"][0]
|
|
assert summary["success"] is False
|
|
assert summary["runs_completed"] == 0
|
|
assert summary["runs_failed"] == 1
|
|
assert "error" in summary
|
|
assert summary["error"] == "All runs failed"
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_exception_in_evaluate_treated_as_failed_run(
|
|
self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
|
|
):
|
|
"""An exception raised by _evaluate_single_configuration counts as a failure."""
|
|
mock_eval.side_effect = RuntimeError("network error")
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Crasher"}],
|
|
output_dir=str(tmp_path),
|
|
)
|
|
|
|
assert result["failed_configurations"] == 1
|
|
# The individual_results entry must contain the error string
|
|
individual = result["results"][0]["individual_results"]
|
|
assert len(individual) == 1
|
|
assert individual[0]["success"] is False
|
|
assert "network error" in individual[0]["error"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. test_evaluate_single_configuration_exception
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestEvaluateSingleConfigurationException:
|
|
"""_evaluate_single_configuration returns a proper error dict on exception."""
|
|
|
|
@patch(f"{MODULE}.SpeedProfiler")
|
|
@patch(f"{MODULE}.ResourceMonitor")
|
|
@patch(f"{MODULE}.get_llm")
|
|
def test_exception_returns_success_false(
|
|
self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
|
|
):
|
|
mock_get_llm.side_effect = ValueError("bad model name")
|
|
mock_profiler = MagicMock()
|
|
mock_profiler_cls.return_value = mock_profiler
|
|
mock_monitor = MagicMock()
|
|
mock_monitor_cls.return_value = mock_monitor
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
|
|
result = _evaluate_single_configuration(
|
|
query="test query", config={"name": "cfg1"}
|
|
)
|
|
|
|
assert result["success"] is False
|
|
|
|
@patch(f"{MODULE}.SpeedProfiler")
|
|
@patch(f"{MODULE}.ResourceMonitor")
|
|
@patch(f"{MODULE}.get_llm")
|
|
def test_exception_message_in_result(
|
|
self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
|
|
):
|
|
mock_get_llm.side_effect = ValueError("bad model name")
|
|
mock_profiler = MagicMock()
|
|
mock_profiler_cls.return_value = mock_profiler
|
|
mock_monitor = MagicMock()
|
|
mock_monitor_cls.return_value = mock_monitor
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
|
|
result = _evaluate_single_configuration(
|
|
query="test query", config={"name": "cfg1"}
|
|
)
|
|
|
|
assert "error" in result
|
|
assert "bad model name" in result["error"]
|
|
|
|
@patch(f"{MODULE}.SpeedProfiler")
|
|
@patch(f"{MODULE}.ResourceMonitor")
|
|
@patch(f"{MODULE}.get_llm")
|
|
def test_exception_result_contains_timing_and_resource_details(
|
|
self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
|
|
):
|
|
"""Even on error the result includes timing_details and resource_details."""
|
|
mock_get_llm.side_effect = Exception("oops")
|
|
|
|
profiler_instance = MagicMock()
|
|
profiler_instance.get_timings.return_value = {"llm_init": 0.5}
|
|
mock_profiler_cls.return_value = profiler_instance
|
|
|
|
monitor_instance = MagicMock()
|
|
monitor_instance.get_combined_stats.return_value = {"cpu": 10.0}
|
|
mock_monitor_cls.return_value = monitor_instance
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
|
|
result = _evaluate_single_configuration(query="q", config={})
|
|
|
|
assert "timing_details" in result
|
|
assert "resource_details" in result
|
|
|
|
@patch(f"{MODULE}.SpeedProfiler")
|
|
@patch(f"{MODULE}.ResourceMonitor")
|
|
@patch(f"{MODULE}.get_llm")
|
|
def test_exception_stops_profiling(
|
|
self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
|
|
):
|
|
"""stop() must be called on both profiler and monitor even on error."""
|
|
mock_get_llm.side_effect = Exception("fail")
|
|
|
|
profiler_instance = MagicMock()
|
|
mock_profiler_cls.return_value = profiler_instance
|
|
|
|
monitor_instance = MagicMock()
|
|
mock_monitor_cls.return_value = monitor_instance
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
|
|
_evaluate_single_configuration(query="q", config={})
|
|
|
|
profiler_instance.stop.assert_called()
|
|
monitor_instance.stop.assert_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. test_calculate_average_metrics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCalculateAverageMetrics:
|
|
"""_calculate_average_metrics averages correctly across all metric types."""
|
|
|
|
def test_quality_metrics_averaged(self):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {"score": 0.4},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {"score": 0.6},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert avg["quality_metrics"]["score"] == pytest.approx(0.5)
|
|
|
|
def test_speed_metrics_averaged(self):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {},
|
|
"speed_metrics": {"total_duration": 30.0},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert avg["speed_metrics"]["total_duration"] == pytest.approx(20.0)
|
|
|
|
def test_resource_metrics_averaged(self):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {"memory_mb": 100.0},
|
|
},
|
|
{
|
|
"quality_metrics": {},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {"memory_mb": 200.0},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert avg["resource_metrics"]["memory_mb"] == pytest.approx(150.0)
|
|
|
|
def test_none_values_filtered_out(self):
|
|
"""None values are skipped so the average uses only real numbers."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {"x": None},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {"x": 0.8},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
# Only the non-None value contributes
|
|
assert avg["quality_metrics"]["x"] == pytest.approx(0.8)
|
|
|
|
def test_disjoint_metric_keys_all_present(self):
|
|
"""Keys present in only one result still appear in the output."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {"key_a": 0.9},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {"key_b": 0.3},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert avg["quality_metrics"]["key_a"] == pytest.approx(0.9)
|
|
assert avg["quality_metrics"]["key_b"] == pytest.approx(0.3)
|
|
|
|
def test_empty_input_returns_empty_dict(self):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
assert _calculate_average_metrics([]) == {}
|
|
|
|
def test_three_results_averaged_correctly(self):
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {"q": 0.2},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {"q": 0.5},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"quality_metrics": {"q": 0.8},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert avg["quality_metrics"]["q"] == pytest.approx(0.5)
|
|
|
|
def test_output_always_has_three_categories(self):
|
|
"""Result always contains quality_metrics, speed_metrics, resource_metrics keys."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"quality_metrics": {"x": 1.0},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
}
|
|
]
|
|
avg = _calculate_average_metrics(results)
|
|
assert "quality_metrics" in avg
|
|
assert "speed_metrics" in avg
|
|
assert "resource_metrics" in avg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. test_create_comparison_visualizations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCreateComparisonVisualizations:
|
|
"""_create_comparison_visualizations makes the expected matplotlib calls."""
|
|
|
|
@patch(f"{MODULE}.plt")
|
|
def test_no_successful_results_skips_all_plots(self, mock_plt, tmp_path):
|
|
"""Early return when there are no successful results."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
|
|
report = {"results": [{"success": False, "error": "failed"}]}
|
|
_create_comparison_visualizations(
|
|
report, output_dir=str(tmp_path), timestamp="20240101_120000"
|
|
)
|
|
|
|
mock_plt.figure.assert_not_called()
|
|
mock_plt.barh.assert_not_called()
|
|
mock_plt.savefig.assert_not_called()
|
|
|
|
@patch(f"{MODULE}._create_spider_chart")
|
|
@patch(f"{MODULE}._create_pareto_chart")
|
|
@patch(f"{MODULE}._create_metric_comparison_chart")
|
|
@patch(f"{MODULE}.plt")
|
|
def test_successful_results_calls_savefig(
|
|
self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path
|
|
):
|
|
"""With one successful result all sub-chart helpers are called."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
|
|
mock_fig = MagicMock()
|
|
mock_plt.figure.return_value = mock_fig
|
|
|
|
report = {
|
|
"results": [
|
|
{
|
|
"success": True,
|
|
"name": "Config A",
|
|
"overall_score": 0.75,
|
|
"avg_metrics": {
|
|
"quality_metrics": {"overall_quality": 0.75},
|
|
"speed_metrics": {"total_duration": 15.0},
|
|
"resource_metrics": {},
|
|
},
|
|
}
|
|
]
|
|
}
|
|
|
|
_create_comparison_visualizations(
|
|
report, output_dir=str(tmp_path), timestamp="20240101_000000"
|
|
)
|
|
|
|
# Overall score bar chart triggers figure/barh/savefig
|
|
mock_plt.figure.assert_called()
|
|
mock_plt.savefig.assert_called()
|
|
mock_plt.close.assert_called()
|
|
|
|
# Sub-chart helpers must have been invoked once each
|
|
mock_metric_chart.assert_called()
|
|
mock_spider.assert_called_once()
|
|
mock_pareto.assert_called_once()
|
|
|
|
@patch(f"{MODULE}._create_spider_chart")
|
|
@patch(f"{MODULE}._create_pareto_chart")
|
|
@patch(f"{MODULE}._create_metric_comparison_chart")
|
|
@patch(f"{MODULE}.plt")
|
|
def test_config_names_extracted_correctly(
|
|
self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path
|
|
):
|
|
"""Config names used in barh must come from result['name']."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
|
|
mock_plt.figure.return_value = MagicMock()
|
|
|
|
report = {
|
|
"results": [
|
|
{
|
|
"success": True,
|
|
"name": "MySpecialConfig",
|
|
"overall_score": 0.5,
|
|
"avg_metrics": {
|
|
"quality_metrics": {},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
},
|
|
}
|
|
]
|
|
}
|
|
|
|
_create_comparison_visualizations(
|
|
report, output_dir=str(tmp_path), timestamp="ts1"
|
|
)
|
|
|
|
# barh should have been called with the name in the y-axis list
|
|
barh_call_args = mock_plt.barh.call_args
|
|
assert barh_call_args is not None
|
|
config_names_arg = barh_call_args[0][0]
|
|
assert "MySpecialConfig" in config_names_arg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. test_comparison_with_repetitions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestComparisonWithRepetitions:
|
|
"""compare_configurations with multiple repetitions averages metrics correctly."""
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}.calculate_combined_score", return_value=0.7)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_three_repetitions_runs_completed(
|
|
self,
|
|
mock_write,
|
|
mock_viz,
|
|
mock_score,
|
|
mock_eval,
|
|
mock_makedirs,
|
|
tmp_path,
|
|
):
|
|
mock_eval.return_value = _SUCCESSFUL_RUN.copy()
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Cfg"}],
|
|
output_dir=str(tmp_path),
|
|
repetitions=3,
|
|
)
|
|
|
|
summary = result["results"][0]
|
|
assert summary["runs_completed"] == 3
|
|
assert summary["runs_failed"] == 0
|
|
assert summary["success"] is True
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}.calculate_combined_score", return_value=0.7)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_partial_repetition_failure(
|
|
self,
|
|
mock_write,
|
|
mock_viz,
|
|
mock_score,
|
|
mock_eval,
|
|
mock_makedirs,
|
|
tmp_path,
|
|
):
|
|
"""Only successful repetitions count toward runs_completed."""
|
|
mock_eval.side_effect = [
|
|
_SUCCESSFUL_RUN.copy(),
|
|
_FAILED_RUN.copy(),
|
|
_SUCCESSFUL_RUN.copy(),
|
|
]
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Cfg"}],
|
|
output_dir=str(tmp_path),
|
|
repetitions=3,
|
|
)
|
|
|
|
summary = result["results"][0]
|
|
assert summary["runs_completed"] == 2
|
|
assert summary["runs_failed"] == 1
|
|
assert summary["success"] is True
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}._calculate_average_metrics")
|
|
@patch(f"{MODULE}.calculate_combined_score", return_value=0.65)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_repetitions_pass_all_successful_runs_to_avg(
|
|
self,
|
|
mock_write,
|
|
mock_viz,
|
|
mock_score,
|
|
mock_avg,
|
|
mock_eval,
|
|
mock_makedirs,
|
|
tmp_path,
|
|
):
|
|
"""_calculate_average_metrics is called with the list of successful runs."""
|
|
run1 = _SUCCESSFUL_RUN.copy()
|
|
run2 = _SUCCESSFUL_RUN.copy()
|
|
mock_eval.side_effect = [run1, run2]
|
|
mock_avg.return_value = {
|
|
"quality_metrics": {},
|
|
"speed_metrics": {},
|
|
"resource_metrics": {},
|
|
}
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Cfg"}],
|
|
output_dir=str(tmp_path),
|
|
repetitions=2,
|
|
)
|
|
|
|
mock_avg.assert_called_once_with([run1, run2])
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}.calculate_combined_score", return_value=0.8)
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_repetitions_field_in_report(
|
|
self,
|
|
mock_write,
|
|
mock_viz,
|
|
mock_score,
|
|
mock_eval,
|
|
mock_makedirs,
|
|
tmp_path,
|
|
):
|
|
"""The top-level 'repetitions' key reflects the requested count."""
|
|
mock_eval.return_value = _SUCCESSFUL_RUN.copy()
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Cfg"}],
|
|
output_dir=str(tmp_path),
|
|
repetitions=5,
|
|
)
|
|
|
|
assert result["repetitions"] == 5
|
|
|
|
@patch(f"{MODULE}.os.makedirs")
|
|
@patch(f"{MODULE}._evaluate_single_configuration")
|
|
@patch(f"{MODULE}.calculate_combined_score")
|
|
@patch(f"{MODULE}._create_comparison_visualizations")
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
def test_two_configs_sorted_by_score(
|
|
self,
|
|
mock_write,
|
|
mock_viz,
|
|
mock_score,
|
|
mock_eval,
|
|
mock_makedirs,
|
|
tmp_path,
|
|
):
|
|
"""Results are sorted descending by overall_score."""
|
|
mock_eval.return_value = _SUCCESSFUL_RUN.copy()
|
|
mock_score.side_effect = [0.3, 0.9]
|
|
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="q",
|
|
configurations=[{"name": "Low"}, {"name": "High"}],
|
|
output_dir=str(tmp_path),
|
|
)
|
|
|
|
scores = [
|
|
r["overall_score"] for r in result["results"] if r.get("success")
|
|
]
|
|
assert scores == sorted(scores, reverse=True)
|