wehub-resource-sync
7a0da7932b
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Has been cancelled
Docker Tests (Consolidated) / Accessibility Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Unit Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Example Tests (push) Has been cancelled
Docker Tests (Consolidated) / Production Image Smoke Test (push) Has been cancelled
Docker Tests (Consolidated) / Infrastructure Tests (push) Has been cancelled
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Has been cancelled
Backwards Compatibility / Verify Encryption Constants (push) Has been cancelled
Backwards Compatibility / PyPI Version Compatibility (push) Has been cancelled
Backwards Compatibility / Database Migration Tests (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Docker Tests (Consolidated) / detect-changes (push) Has been cancelled
Docker Tests (Consolidated) / Build Test Image (push) Has been cancelled
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Has been cancelled
923 lines
31 KiB
Python
923 lines
31 KiB
Python
"""
|
|
Tests for benchmarks/comparison/evaluator.py
|
|
|
|
Tests cover:
|
|
- compare_configurations function
|
|
- _calculate_average_metrics function
|
|
- _evaluate_single_configuration function structure
|
|
- Visualization creation functions
|
|
"""
|
|
|
|
from unittest.mock import patch
|
|
import pytest
|
|
|
|
|
|
class TestCompareConfigurationsValidation:
|
|
"""Tests for compare_configurations input validation."""
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_returns_error_for_empty_configurations(
|
|
self, mock_makedirs, tmp_path
|
|
):
|
|
"""Test that empty configurations list returns error."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="test query",
|
|
configurations=[],
|
|
output_dir=str(tmp_path),
|
|
)
|
|
|
|
assert "error" in result
|
|
assert result["error"] == "No configurations provided"
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_creates_output_directory(self, mock_makedirs):
|
|
"""Test that output directory is created."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
# Empty configs will return early with error but still create dir
|
|
compare_configurations(
|
|
query="test",
|
|
configurations=[],
|
|
output_dir="/tmp/test_output",
|
|
)
|
|
|
|
mock_makedirs.assert_called_with("/tmp/test_output", exist_ok=True)
|
|
|
|
|
|
class TestCalculateAverageMetrics:
|
|
"""Tests for _calculate_average_metrics function."""
|
|
|
|
def test_calculates_average_quality_metrics(self):
|
|
"""Test averaging of quality metrics."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{"quality_metrics": {"accuracy": 0.8, "source_count": 10}},
|
|
{"quality_metrics": {"accuracy": 0.6, "source_count": 8}},
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
assert avg["quality_metrics"]["accuracy"] == pytest.approx(0.7)
|
|
assert avg["quality_metrics"]["source_count"] == pytest.approx(9.0)
|
|
|
|
def test_calculates_average_speed_metrics(self):
|
|
"""Test averaging of speed metrics."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{
|
|
"speed_metrics": {
|
|
"total_duration": 100,
|
|
"duration_per_question": 10,
|
|
}
|
|
},
|
|
{
|
|
"speed_metrics": {
|
|
"total_duration": 80,
|
|
"duration_per_question": 8,
|
|
}
|
|
},
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
assert avg["speed_metrics"]["total_duration"] == pytest.approx(90.0)
|
|
assert avg["speed_metrics"]["duration_per_question"] == pytest.approx(
|
|
9.0
|
|
)
|
|
|
|
def test_calculates_average_resource_metrics(self):
|
|
"""Test averaging of resource metrics."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{"resource_metrics": {"memory_mb": 100}},
|
|
{"resource_metrics": {"memory_mb": 200}},
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
assert avg["resource_metrics"]["memory_mb"] == pytest.approx(150.0)
|
|
|
|
def test_handles_empty_results(self):
|
|
"""Test handling of empty results list."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
avg = _calculate_average_metrics([])
|
|
|
|
assert avg == {}
|
|
|
|
def test_handles_missing_metrics_categories(self):
|
|
"""Test handling of results with missing metric categories."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{"quality_metrics": {"accuracy": 0.8}},
|
|
{}, # No metrics
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
# Should still have quality_metrics
|
|
assert "quality_metrics" in avg
|
|
assert avg["quality_metrics"]["accuracy"] == pytest.approx(0.8)
|
|
|
|
def test_handles_none_values(self):
|
|
"""Test handling of None values in metrics."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{"quality_metrics": {"accuracy": 0.8, "optional": None}},
|
|
{"quality_metrics": {"accuracy": 0.6, "optional": 0.5}},
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
# None values should be filtered out
|
|
assert avg["quality_metrics"]["accuracy"] == pytest.approx(0.7)
|
|
# Optional should only have one valid value
|
|
assert avg["quality_metrics"]["optional"] == pytest.approx(0.5)
|
|
|
|
def test_handles_mixed_metric_keys(self):
|
|
"""Test handling when results have different metric keys."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_calculate_average_metrics,
|
|
)
|
|
|
|
results = [
|
|
{"quality_metrics": {"metric_a": 0.8}},
|
|
{"quality_metrics": {"metric_b": 0.6}},
|
|
]
|
|
|
|
avg = _calculate_average_metrics(results)
|
|
|
|
# Both metrics should be present
|
|
assert avg["quality_metrics"]["metric_a"] == pytest.approx(0.8)
|
|
assert avg["quality_metrics"]["metric_b"] == pytest.approx(0.6)
|
|
|
|
|
|
class TestMetricWeightsDefault:
|
|
"""Tests for default metric weights."""
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_default_metric_weights(self, mock_makedirs):
|
|
"""Test that default metric weights are applied."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
compare_configurations(
|
|
query="test",
|
|
configurations=[],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
# Empty configs returns early, but we can verify the function signature
|
|
assert callable(compare_configurations)
|
|
|
|
|
|
class TestVisualizationFunctions:
|
|
"""Tests for visualization helper functions."""
|
|
|
|
def test_create_metric_comparison_chart_exists(self):
|
|
"""Test that _create_metric_comparison_chart function exists."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_metric_comparison_chart,
|
|
)
|
|
|
|
assert callable(_create_metric_comparison_chart)
|
|
|
|
def test_create_spider_chart_exists(self):
|
|
"""Test that _create_spider_chart function exists."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_spider_chart,
|
|
)
|
|
|
|
assert callable(_create_spider_chart)
|
|
|
|
def test_create_pareto_chart_exists(self):
|
|
"""Test that _create_pareto_chart function exists."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_pareto_chart,
|
|
)
|
|
|
|
assert callable(_create_pareto_chart)
|
|
|
|
def test_create_comparison_visualizations_exists(self):
|
|
"""Test that _create_comparison_visualizations function exists."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
|
|
assert callable(_create_comparison_visualizations)
|
|
|
|
|
|
class TestEvaluateSingleConfiguration:
|
|
"""Tests for _evaluate_single_configuration function."""
|
|
|
|
def test_evaluate_single_configuration_exists(self):
|
|
"""Test that _evaluate_single_configuration function exists."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
|
|
assert callable(_evaluate_single_configuration)
|
|
|
|
def test_function_signature(self):
|
|
"""Test function has expected parameters."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
import inspect
|
|
|
|
sig = inspect.signature(_evaluate_single_configuration)
|
|
params = list(sig.parameters.keys())
|
|
|
|
assert "query" in params
|
|
assert "config" in params
|
|
|
|
|
|
class TestConfigurationResultStructure:
|
|
"""Tests for expected result structure."""
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_returns_dict(self, mock_makedirs):
|
|
"""Test that compare_configurations returns a dictionary."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
assert isinstance(result, dict)
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_error_result_has_error_key(self, mock_makedirs):
|
|
"""Test that error results have 'error' key."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
assert "error" in result
|
|
|
|
|
|
class TestCompareConfigurationsWithMocks:
|
|
"""Tests for compare_configurations with full mocking."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_single_configuration(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test comparing a single configuration."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.return_value = 0.8
|
|
mock_eval.return_value = {
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
}
|
|
|
|
result = compare_configurations(
|
|
query="test query",
|
|
configurations=[{"name": "Config 1", "iterations": 2}],
|
|
output_dir="/tmp/test",
|
|
repetitions=1,
|
|
)
|
|
|
|
assert result["configurations_tested"] == 1
|
|
assert result["successful_configurations"] == 1
|
|
assert len(result["results"]) == 1
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_multiple_configurations(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test comparing multiple configurations."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.return_value = 0.8
|
|
mock_eval.side_effect = [
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.7},
|
|
"speed_metrics": {"total_duration": 15.0},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[
|
|
{"name": "Config 1"},
|
|
{"name": "Config 2"},
|
|
],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
assert result["configurations_tested"] == 2
|
|
assert result["successful_configurations"] == 2
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_handles_failed_configuration(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test handling of failed configuration."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.return_value = 0.0
|
|
mock_eval.return_value = {
|
|
"success": False,
|
|
"error": "Config failed",
|
|
}
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[{"name": "Failing Config"}],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
assert result["failed_configurations"] == 1
|
|
assert result["successful_configurations"] == 0
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_with_multiple_repetitions(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test compare with multiple repetitions per configuration."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.return_value = 0.8
|
|
# Three repetitions for one config
|
|
mock_eval.side_effect = [
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.85},
|
|
"speed_metrics": {"total_duration": 9.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.75},
|
|
"speed_metrics": {"total_duration": 11.0},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[{"name": "Config 1"}],
|
|
output_dir="/tmp/test",
|
|
repetitions=3,
|
|
)
|
|
|
|
assert result["repetitions"] == 3
|
|
assert result["results"][0]["runs_completed"] == 3
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_compare_custom_metric_weights(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test compare with custom metric weights."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.return_value = 0.8
|
|
mock_eval.return_value = {
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
}
|
|
|
|
custom_weights = {"quality": 0.8, "speed": 0.2, "resource": 0.0}
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[{"name": "Config 1"}],
|
|
output_dir="/tmp/test",
|
|
metric_weights=custom_weights,
|
|
)
|
|
|
|
assert result["metric_weights"] == custom_weights
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._evaluate_single_configuration"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator._create_comparison_visualizations"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_combined_score"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.os.makedirs")
|
|
def test_results_sorted_by_score_descending(
|
|
self, mock_makedirs, mock_write, mock_combined, mock_viz, mock_eval
|
|
):
|
|
"""Test that results are sorted by score in descending order."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
compare_configurations,
|
|
)
|
|
|
|
mock_combined.side_effect = [0.5, 0.9, 0.7]
|
|
mock_eval.side_effect = [
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.5},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.9},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
{
|
|
"success": True,
|
|
"quality_metrics": {"overall_quality": 0.7},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
]
|
|
|
|
result = compare_configurations(
|
|
query="test",
|
|
configurations=[
|
|
{"name": "Low"},
|
|
{"name": "High"},
|
|
{"name": "Mid"},
|
|
],
|
|
output_dir="/tmp/test",
|
|
)
|
|
|
|
# Successful results should be sorted by score descending
|
|
successful = [r for r in result["results"] if r.get("success")]
|
|
scores = [r.get("overall_score", 0) for r in successful]
|
|
assert scores == sorted(scores, reverse=True)
|
|
|
|
|
|
class TestEvaluateSingleConfigurationFull:
|
|
"""Full tests for _evaluate_single_configuration function."""
|
|
|
|
from unittest.mock import Mock
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.get_llm")
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.get_search")
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.AdvancedSearchSystem"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.SpeedProfiler")
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.ResourceMonitor"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_quality_metrics"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_speed_metrics"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_resource_metrics"
|
|
)
|
|
def test_successful_evaluation(
|
|
self,
|
|
mock_res_metrics,
|
|
mock_speed_metrics,
|
|
mock_quality_metrics,
|
|
mock_res_monitor,
|
|
mock_profiler,
|
|
mock_search_system,
|
|
mock_get_search,
|
|
mock_get_llm,
|
|
):
|
|
"""Test successful configuration evaluation."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
from unittest.mock import Mock
|
|
|
|
# Setup mocks
|
|
mock_llm = Mock()
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
mock_search = Mock()
|
|
mock_get_search.return_value = mock_search
|
|
|
|
mock_system = Mock()
|
|
mock_system.analyze_topic.return_value = {
|
|
"findings": [{"phase": 1, "content": "test"}],
|
|
"current_knowledge": "Test knowledge",
|
|
}
|
|
mock_system.all_links_of_system = ["http://example.com"]
|
|
mock_search_system.return_value = mock_system
|
|
|
|
mock_profiler_instance = Mock()
|
|
mock_profiler_instance.timer.return_value.__enter__ = Mock()
|
|
mock_profiler_instance.timer.return_value.__exit__ = Mock(
|
|
return_value=False
|
|
)
|
|
mock_profiler_instance.get_summary.return_value = {}
|
|
mock_profiler_instance.get_timings.return_value = {}
|
|
mock_profiler.return_value = mock_profiler_instance
|
|
|
|
mock_res_monitor_instance = Mock()
|
|
mock_res_monitor_instance.get_combined_stats.return_value = {}
|
|
mock_res_monitor.return_value = mock_res_monitor_instance
|
|
|
|
mock_quality_metrics.return_value = {"overall_quality": 0.8}
|
|
mock_speed_metrics.return_value = {"total_duration": 10.0}
|
|
mock_res_metrics.return_value = {}
|
|
|
|
config = {"iterations": 2, "search_strategy": "iterdrag"}
|
|
|
|
result = _evaluate_single_configuration(
|
|
query="test query",
|
|
config=config,
|
|
)
|
|
|
|
assert result["success"] is True
|
|
assert "quality_metrics" in result
|
|
assert "speed_metrics" in result
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.get_llm")
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.SpeedProfiler")
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.ResourceMonitor"
|
|
)
|
|
def test_evaluation_handles_llm_error(
|
|
self,
|
|
mock_res_monitor,
|
|
mock_profiler,
|
|
mock_get_llm,
|
|
):
|
|
"""Test that evaluation handles LLM initialization errors."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
from unittest.mock import Mock
|
|
|
|
mock_get_llm.side_effect = Exception("LLM init failed")
|
|
|
|
mock_profiler_instance = Mock()
|
|
mock_profiler_instance.timer.return_value.__enter__ = Mock()
|
|
mock_profiler_instance.timer.return_value.__exit__ = Mock(
|
|
return_value=False
|
|
)
|
|
mock_profiler_instance.get_timings.return_value = {}
|
|
mock_profiler.return_value = mock_profiler_instance
|
|
|
|
mock_res_monitor_instance = Mock()
|
|
mock_res_monitor_instance.get_combined_stats.return_value = {}
|
|
mock_res_monitor.return_value = mock_res_monitor_instance
|
|
|
|
config = {"iterations": 2}
|
|
|
|
result = _evaluate_single_configuration(
|
|
query="test",
|
|
config=config,
|
|
)
|
|
|
|
assert result["success"] is False
|
|
assert "error" in result
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.get_llm")
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.get_search")
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.AdvancedSearchSystem"
|
|
)
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.SpeedProfiler")
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.ResourceMonitor"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_quality_metrics"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_speed_metrics"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.comparison.evaluator.calculate_resource_metrics"
|
|
)
|
|
def test_evaluation_uses_config_parameters(
|
|
self,
|
|
mock_res_metrics,
|
|
mock_speed_metrics,
|
|
mock_quality_metrics,
|
|
mock_res_monitor,
|
|
mock_profiler,
|
|
mock_search_system,
|
|
mock_get_search,
|
|
mock_get_llm,
|
|
):
|
|
"""Test that configuration parameters are applied correctly."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_evaluate_single_configuration,
|
|
)
|
|
from unittest.mock import Mock
|
|
|
|
mock_llm = Mock()
|
|
mock_get_llm.return_value = mock_llm
|
|
|
|
mock_search = Mock()
|
|
mock_get_search.return_value = mock_search
|
|
|
|
mock_system = Mock()
|
|
mock_system.analyze_topic.return_value = {
|
|
"findings": [],
|
|
"current_knowledge": "",
|
|
}
|
|
mock_search_system.return_value = mock_system
|
|
|
|
mock_profiler_instance = Mock()
|
|
mock_profiler_instance.timer.return_value.__enter__ = Mock()
|
|
mock_profiler_instance.timer.return_value.__exit__ = Mock(
|
|
return_value=False
|
|
)
|
|
mock_profiler_instance.get_summary.return_value = {}
|
|
mock_profiler_instance.get_timings.return_value = {}
|
|
mock_profiler.return_value = mock_profiler_instance
|
|
|
|
mock_res_monitor_instance = Mock()
|
|
mock_res_monitor_instance.get_combined_stats.return_value = {}
|
|
mock_res_monitor.return_value = mock_res_monitor_instance
|
|
|
|
mock_quality_metrics.return_value = {}
|
|
mock_speed_metrics.return_value = {}
|
|
mock_res_metrics.return_value = {}
|
|
|
|
config = {
|
|
"iterations": 5,
|
|
"questions_per_iteration": 4,
|
|
"search_strategy": "focused_iteration",
|
|
}
|
|
|
|
_evaluate_single_configuration(
|
|
query="test",
|
|
config=config,
|
|
)
|
|
|
|
# Verify system was configured with our parameters
|
|
call_kwargs = mock_search_system.call_args[1]
|
|
assert call_kwargs["max_iterations"] == 5
|
|
assert call_kwargs["questions_per_iteration"] == 4
|
|
assert call_kwargs["strategy_name"] == "focused_iteration"
|
|
|
|
|
|
class TestVisualizationCreation:
|
|
"""Tests for visualization creation functions."""
|
|
|
|
from unittest.mock import Mock
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.plt")
|
|
def test_create_comparison_visualizations_no_successful(self, mock_plt):
|
|
"""Test visualization with no successful results."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
|
|
report = {"results": [{"success": False}]}
|
|
|
|
# Should not raise
|
|
_create_comparison_visualizations(
|
|
report, output_dir="/tmp/test", timestamp="20240101"
|
|
)
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.plt")
|
|
def test_create_metric_comparison_chart_single_metric(self, mock_plt):
|
|
"""Test metric comparison chart with single metric."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_metric_comparison_chart,
|
|
)
|
|
from unittest.mock import Mock, MagicMock
|
|
|
|
mock_fig = Mock()
|
|
mock_ax = MagicMock()
|
|
mock_plt.subplots.return_value = (mock_fig, mock_ax)
|
|
|
|
# barh returns a BarContainer which is iterable; mock it with a list
|
|
# of mock bars that support get_width/get_y/get_height
|
|
mock_bar = Mock()
|
|
mock_bar.get_width.return_value = 0.8
|
|
mock_bar.get_y.return_value = 0.0
|
|
mock_bar.get_height.return_value = 0.5
|
|
mock_ax.barh.return_value = [mock_bar]
|
|
|
|
results = [
|
|
{
|
|
"name": "Config 1",
|
|
"avg_metrics": {"quality_metrics": {"overall_quality": 0.8}},
|
|
}
|
|
]
|
|
|
|
_create_metric_comparison_chart(
|
|
results,
|
|
["Config 1"],
|
|
["overall_quality"],
|
|
"quality_metrics",
|
|
"Test",
|
|
"/tmp/test.png",
|
|
)
|
|
|
|
mock_plt.savefig.assert_called()
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.plt")
|
|
def test_create_pareto_chart_with_data(self, mock_plt):
|
|
"""Test pareto chart creation with data."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_pareto_chart,
|
|
)
|
|
from unittest.mock import Mock
|
|
|
|
mock_fig = Mock()
|
|
mock_ax = Mock()
|
|
mock_plt.subplots.return_value = (mock_fig, mock_ax)
|
|
mock_plt.figure.return_value = mock_fig
|
|
|
|
results = [
|
|
{
|
|
"name": "Config 1",
|
|
"avg_metrics": {
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
},
|
|
},
|
|
{
|
|
"name": "Config 2",
|
|
"avg_metrics": {
|
|
"quality_metrics": {"overall_quality": 0.6},
|
|
"speed_metrics": {"total_duration": 5.0},
|
|
},
|
|
},
|
|
]
|
|
|
|
_create_pareto_chart(results, "/tmp/pareto.png")
|
|
|
|
mock_plt.savefig.assert_called()
|
|
|
|
@patch("local_deep_research.benchmarks.comparison.evaluator.plt")
|
|
def test_create_comparison_visualizations_creates_files(self, mock_plt):
|
|
"""Test that visualizations create output files."""
|
|
from local_deep_research.benchmarks.comparison.evaluator import (
|
|
_create_comparison_visualizations,
|
|
)
|
|
from unittest.mock import Mock, MagicMock
|
|
|
|
def _make_mock_ax():
|
|
"""Create a mock ax that supports barh returning iterable bars."""
|
|
ax = MagicMock()
|
|
bar = Mock()
|
|
bar.get_width.return_value = 0.5
|
|
bar.get_y.return_value = 0.0
|
|
bar.get_height.return_value = 0.5
|
|
ax.barh.return_value = [bar]
|
|
return ax
|
|
|
|
mock_fig = Mock()
|
|
|
|
def _subplots_side_effect(*args, **kwargs):
|
|
"""Return (fig, axes) with correct number of axes."""
|
|
nrows = args[0] if args else kwargs.get("nrows", 1)
|
|
if nrows == 1:
|
|
return (mock_fig, _make_mock_ax())
|
|
return (mock_fig, [_make_mock_ax() for _ in range(nrows)])
|
|
|
|
mock_plt.subplots.side_effect = _subplots_side_effect
|
|
mock_plt.figure.return_value = mock_fig
|
|
|
|
report = {
|
|
"results": [
|
|
{
|
|
"name": "Config 1",
|
|
"success": True,
|
|
"overall_score": 0.8,
|
|
"avg_metrics": {
|
|
"quality_metrics": {"overall_quality": 0.8},
|
|
"speed_metrics": {"total_duration": 10.0},
|
|
"resource_metrics": {},
|
|
},
|
|
}
|
|
]
|
|
}
|
|
|
|
_create_comparison_visualizations(
|
|
report, output_dir="/tmp/test", timestamp="20240101"
|
|
)
|
|
|
|
# Should have called savefig multiple times
|
|
assert mock_plt.savefig.called
|