wehub-resource-sync
7a0da7932b
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Has been cancelled
Docker Tests (Consolidated) / Accessibility Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Unit Tests (push) Has been cancelled
Docker Tests (Consolidated) / LLM Example Tests (push) Has been cancelled
Docker Tests (Consolidated) / Production Image Smoke Test (push) Has been cancelled
Docker Tests (Consolidated) / Infrastructure Tests (push) Has been cancelled
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Has been cancelled
Backwards Compatibility / Verify Encryption Constants (push) Has been cancelled
Backwards Compatibility / PyPI Version Compatibility (push) Has been cancelled
Backwards Compatibility / Database Migration Tests (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Docker Tests (Consolidated) / detect-changes (push) Has been cancelled
Docker Tests (Consolidated) / Build Test Image (push) Has been cancelled
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Has been cancelled
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Has been cancelled
1021 lines
34 KiB
Python
1021 lines
34 KiB
Python
"""
|
|
Tests for benchmarks/optimization/optuna_optimizer.py
|
|
|
|
Tests cover:
|
|
- OptunaOptimizer initialization
|
|
- Default parameter space
|
|
- Weight normalization
|
|
- Convenience optimization functions
|
|
"""
|
|
|
|
from unittest.mock import Mock, patch
|
|
import pytest
|
|
|
|
|
|
class TestOptunaOptimizerInit:
|
|
"""Tests for OptunaOptimizer initialization."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_with_defaults(self, mock_evaluator):
|
|
"""Test initialization with default values."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test query",
|
|
output_dir="/tmp/test_output",
|
|
)
|
|
|
|
assert optimizer.base_query == "test query"
|
|
assert optimizer.output_dir == "/tmp/test_output"
|
|
assert optimizer.n_trials == 30
|
|
assert optimizer.n_jobs == 1
|
|
assert optimizer.temperature == 0.7
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_with_custom_values(self, mock_evaluator):
|
|
"""Test initialization with custom values."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test query",
|
|
output_dir="/tmp/test",
|
|
model_name="custom-model",
|
|
provider="openai",
|
|
search_tool="google",
|
|
temperature=0.5,
|
|
n_trials=50,
|
|
n_jobs=4,
|
|
)
|
|
|
|
assert optimizer.model_name == "custom-model"
|
|
assert optimizer.provider == "openai"
|
|
assert optimizer.search_tool == "google"
|
|
assert optimizer.temperature == 0.5
|
|
assert optimizer.n_trials == 50
|
|
assert optimizer.n_jobs == 4
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_normalizes_weights(self, mock_evaluator):
|
|
"""Test that metric weights are normalized."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
metric_weights={"quality": 2, "speed": 2},
|
|
)
|
|
|
|
# Weights should be normalized to sum to 1
|
|
total = sum(optimizer.metric_weights.values())
|
|
assert total == pytest.approx(1.0)
|
|
assert optimizer.metric_weights["quality"] == pytest.approx(0.5)
|
|
assert optimizer.metric_weights["speed"] == pytest.approx(0.5)
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_default_benchmark_weights(self, mock_evaluator):
|
|
"""Test default benchmark weights."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert "simpleqa" in optimizer.benchmark_weights
|
|
assert optimizer.benchmark_weights["simpleqa"] == 1.0
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_custom_benchmark_weights(self, mock_evaluator):
|
|
"""Test custom benchmark weights."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
benchmark_weights={"simpleqa": 0.6, "browsecomp": 0.4},
|
|
)
|
|
|
|
assert optimizer.benchmark_weights["simpleqa"] == 0.6
|
|
assert optimizer.benchmark_weights["browsecomp"] == 0.4
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_generates_study_name(self, mock_evaluator):
|
|
"""Test that study name is generated if not provided."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert optimizer.study_name.startswith("ldr_opt_")
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_init_uses_custom_study_name(self, mock_evaluator):
|
|
"""Test that custom study name is used."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
study_name="my_custom_study",
|
|
)
|
|
|
|
assert optimizer.study_name == "my_custom_study"
|
|
|
|
|
|
class TestDefaultParamSpace:
|
|
"""Tests for default parameter space."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_get_default_param_space(self, mock_evaluator):
|
|
"""Test getting default parameter space."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
param_space = optimizer._get_default_param_space()
|
|
|
|
assert "iterations" in param_space
|
|
assert "questions_per_iteration" in param_space
|
|
assert "search_strategy" in param_space
|
|
assert "max_results" in param_space
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_iterations_param_space(self, mock_evaluator):
|
|
"""Test iterations parameter space."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
param_space = optimizer._get_default_param_space()
|
|
|
|
iterations = param_space["iterations"]
|
|
assert iterations["type"] == "int"
|
|
assert iterations["low"] == 1
|
|
assert iterations["high"] == 5
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_search_strategy_param_space(self, mock_evaluator):
|
|
"""Test search strategy parameter space."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
param_space = optimizer._get_default_param_space()
|
|
|
|
strategy = param_space["search_strategy"]
|
|
assert strategy["type"] == "categorical"
|
|
assert "choices" in strategy
|
|
assert "source-based" in strategy["choices"]
|
|
|
|
|
|
class TestConvenienceFunctions:
|
|
"""Tests for convenience optimization functions."""
|
|
|
|
def test_optimize_parameters_exists(self):
|
|
"""Test that optimize_parameters function exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_parameters,
|
|
)
|
|
|
|
assert callable(optimize_parameters)
|
|
|
|
def test_optimize_for_speed_exists(self):
|
|
"""Test that optimize_for_speed function exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_speed,
|
|
)
|
|
|
|
assert callable(optimize_for_speed)
|
|
|
|
def test_optimize_for_quality_exists(self):
|
|
"""Test that optimize_for_quality function exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_quality,
|
|
)
|
|
|
|
assert callable(optimize_for_quality)
|
|
|
|
def test_optimize_for_efficiency_exists(self):
|
|
"""Test that optimize_for_efficiency function exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_efficiency,
|
|
)
|
|
|
|
assert callable(optimize_for_efficiency)
|
|
|
|
|
|
class TestOptimizeFunctionSignatures:
|
|
"""Tests for optimization function signatures."""
|
|
|
|
def test_optimize_for_speed_default_weights(self):
|
|
"""Test optimize_for_speed uses speed-focused weights."""
|
|
import inspect
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_speed,
|
|
)
|
|
|
|
# Check function source for speed weights
|
|
source = inspect.getsource(optimize_for_speed)
|
|
assert "speed" in source.lower()
|
|
|
|
def test_optimize_for_quality_default_weights(self):
|
|
"""Test optimize_for_quality uses quality-focused weights."""
|
|
import inspect
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_quality,
|
|
)
|
|
|
|
source = inspect.getsource(optimize_for_quality)
|
|
assert "quality" in source.lower()
|
|
|
|
def test_optimize_for_efficiency_default_weights(self):
|
|
"""Test optimize_for_efficiency uses balanced weights."""
|
|
import inspect
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_efficiency,
|
|
)
|
|
|
|
source = inspect.getsource(optimize_for_efficiency)
|
|
assert "resource" in source.lower()
|
|
|
|
|
|
class TestOptimizerState:
|
|
"""Tests for optimizer state management."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_initial_state(self, mock_evaluator):
|
|
"""Test initial optimizer state."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert optimizer.best_params is None
|
|
assert optimizer.study is None
|
|
assert optimizer.trials_history == []
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_optimizer_stores_progress_callback(self, mock_evaluator):
|
|
"""Test that progress callback is stored."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
callback = Mock()
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
progress_callback=callback,
|
|
)
|
|
|
|
assert optimizer.progress_callback is callback
|
|
|
|
|
|
class TestPlottingAvailability:
|
|
"""Tests for plotting availability handling."""
|
|
|
|
def test_plotting_available_flag_exists(self):
|
|
"""Test that PLOTTING_AVAILABLE flag exists."""
|
|
from local_deep_research.benchmarks.optimization import optuna_optimizer
|
|
|
|
assert hasattr(optuna_optimizer, "PLOTTING_AVAILABLE")
|
|
assert isinstance(optuna_optimizer.PLOTTING_AVAILABLE, bool)
|
|
|
|
|
|
class TestObjectiveFunction:
|
|
"""Tests for the objective function."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_objective_method_exists(self, mock_evaluator):
|
|
"""Test that _objective method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_objective")
|
|
assert callable(optimizer._objective)
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_run_experiment_method_exists(self, mock_evaluator):
|
|
"""Test that _run_experiment method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_run_experiment")
|
|
assert callable(optimizer._run_experiment)
|
|
|
|
|
|
class TestVisualizationMethods:
|
|
"""Tests for visualization methods."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_create_visualizations_method_exists(self, mock_evaluator):
|
|
"""Test that _create_visualizations method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_create_visualizations")
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_create_quick_visualizations_method_exists(self, mock_evaluator):
|
|
"""Test that _create_quick_visualizations method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_create_quick_visualizations")
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_save_results_method_exists(self, mock_evaluator):
|
|
"""Test that _save_results method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_save_results")
|
|
|
|
|
|
class TestOptimizeMethod:
|
|
"""Tests for the optimize method."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.optuna"
|
|
)
|
|
def test_optimize_creates_study(self, mock_optuna, mock_evaluator):
|
|
"""Test that optimize creates an Optuna study."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
mock_study = Mock()
|
|
mock_study.best_params = {"iterations": 2}
|
|
mock_study.best_value = 0.8
|
|
mock_study.best_trial = Mock()
|
|
mock_study.best_trial.user_attrs = {}
|
|
mock_study.trials = []
|
|
mock_optuna.create_study.return_value = mock_study
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test query",
|
|
n_trials=1,
|
|
)
|
|
|
|
# Mock _save_results to avoid file operations
|
|
with patch.object(optimizer, "_save_results"):
|
|
with patch.object(optimizer, "_create_visualizations"):
|
|
optimizer.optimize()
|
|
|
|
mock_optuna.create_study.assert_called_once()
|
|
assert optimizer.study == mock_study
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.optuna"
|
|
)
|
|
def test_optimize_returns_best_params(self, mock_optuna, mock_evaluator):
|
|
"""Test that optimize returns best parameters."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
mock_study = Mock()
|
|
mock_study.best_params = {"iterations": 3, "questions_per_iteration": 4}
|
|
mock_study.best_value = 0.85
|
|
mock_study.best_trial = Mock()
|
|
mock_study.best_trial.user_attrs = {}
|
|
mock_study.trials = []
|
|
mock_optuna.create_study.return_value = mock_study
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
n_trials=1,
|
|
)
|
|
|
|
with patch.object(optimizer, "_save_results"):
|
|
with patch.object(optimizer, "_create_visualizations"):
|
|
best_params, best_value = optimizer.optimize()
|
|
|
|
assert isinstance(best_params, dict)
|
|
assert best_params["iterations"] == 3
|
|
assert best_value == 0.85
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.optuna"
|
|
)
|
|
def test_optimize_stores_trials_history(self, mock_optuna, mock_evaluator):
|
|
"""Test that optimize stores trials history."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
# Create mock trials
|
|
mock_trial1 = Mock()
|
|
mock_trial1.params = {"iterations": 2}
|
|
mock_trial1.value = 0.7
|
|
mock_trial1.user_attrs = {}
|
|
|
|
mock_trial2 = Mock()
|
|
mock_trial2.params = {"iterations": 3}
|
|
mock_trial2.value = 0.8
|
|
mock_trial2.user_attrs = {}
|
|
|
|
mock_study = Mock()
|
|
mock_study.best_params = {"iterations": 3}
|
|
mock_study.best_value = 0.8
|
|
mock_study.best_trial = mock_trial2
|
|
mock_study.trials = [mock_trial1, mock_trial2]
|
|
mock_optuna.create_study.return_value = mock_study
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
n_trials=2,
|
|
)
|
|
|
|
with patch.object(optimizer, "_save_results"):
|
|
with patch.object(optimizer, "_create_visualizations"):
|
|
optimizer.optimize()
|
|
|
|
# Trials history should be populated from the study callback
|
|
assert optimizer.study is not None
|
|
|
|
|
|
class TestObjectiveFunctionExecution:
|
|
"""Tests for objective function execution."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_objective_suggests_parameters(self, mock_evaluator):
|
|
"""Test that objective function suggests parameters from trial."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
# Create a mock trial
|
|
mock_trial = Mock()
|
|
mock_trial.suggest_int.return_value = 2
|
|
mock_trial.suggest_float.return_value = 0.7
|
|
mock_trial.suggest_categorical.return_value = "iterdrag"
|
|
mock_trial.set_user_attr = Mock()
|
|
|
|
# Mock _run_experiment to return a score
|
|
with patch.object(optimizer, "_run_experiment") as mock_run:
|
|
mock_run.return_value = {
|
|
"score": 0.75,
|
|
"quality_score": 0.8,
|
|
"speed_score": 0.7,
|
|
}
|
|
|
|
param_space = optimizer._get_default_param_space()
|
|
score = optimizer._objective(mock_trial, param_space=param_space)
|
|
|
|
assert score == 0.75
|
|
mock_trial.suggest_int.assert_called()
|
|
mock_trial.suggest_categorical.assert_called()
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_objective_handles_experiment_error(self, mock_evaluator):
|
|
"""Test that objective handles experiment errors gracefully."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
mock_trial = Mock()
|
|
mock_trial.suggest_int.return_value = 2
|
|
mock_trial.suggest_float.return_value = 0.7
|
|
mock_trial.suggest_categorical.return_value = "iterdrag"
|
|
mock_trial.set_user_attr = Mock()
|
|
|
|
# Mock _run_experiment to raise an exception
|
|
with patch.object(optimizer, "_run_experiment") as mock_run:
|
|
mock_run.side_effect = Exception("Experiment failed")
|
|
|
|
param_space = optimizer._get_default_param_space()
|
|
score = optimizer._objective(mock_trial, param_space=param_space)
|
|
|
|
# Should return -inf on error (worst possible score for maximization)
|
|
assert score == float("-inf")
|
|
|
|
|
|
class TestRunExperiment:
|
|
"""Tests for run experiment functionality."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.SpeedProfiler"
|
|
)
|
|
def test_run_experiment_calculates_score(
|
|
self, mock_profiler, mock_evaluator
|
|
):
|
|
"""Test that run_experiment calculates weighted score."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
# Setup mock evaluator - evaluate() returns dict with "quality_score"
|
|
mock_eval_instance = Mock()
|
|
mock_eval_instance.evaluate.return_value = {
|
|
"quality_score": 0.8,
|
|
"benchmark_results": {},
|
|
}
|
|
mock_evaluator.return_value = mock_eval_instance
|
|
|
|
# Setup mock profiler - source calls start(), stop(), get_summary()
|
|
mock_profiler_instance = Mock()
|
|
mock_profiler_instance.get_summary.return_value = {
|
|
"total_duration": 10.0,
|
|
}
|
|
mock_profiler.return_value = mock_profiler_instance
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
metric_weights={"quality": 0.7, "speed": 0.3},
|
|
)
|
|
|
|
params = {
|
|
"iterations": 2,
|
|
"questions_per_iteration": 3,
|
|
"search_strategy": "iterdrag",
|
|
"max_results": 50,
|
|
}
|
|
|
|
result = optimizer._run_experiment(params)
|
|
|
|
assert "score" in result
|
|
assert "quality_score" in result
|
|
assert "speed_score" in result
|
|
assert result["success"] is True
|
|
|
|
|
|
class TestSaveResults:
|
|
"""Tests for save results functionality."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.joblib"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_save_results_creates_json(
|
|
self, mock_evaluator, mock_write_json, mock_joblib
|
|
):
|
|
"""Test that _save_results creates JSON output."""
|
|
import tempfile
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
output_dir=tmpdir,
|
|
)
|
|
|
|
# Setup mock study
|
|
mock_study = Mock()
|
|
mock_study.best_params = {"iterations": 2}
|
|
mock_study.best_value = 0.8
|
|
mock_study.best_trial = Mock()
|
|
mock_study.best_trial.user_attrs = {}
|
|
mock_study.trials = [Mock()]
|
|
optimizer.study = mock_study
|
|
optimizer.best_params = {"iterations": 2}
|
|
optimizer.trials_history = [
|
|
{"params": {"iterations": 2}, "score": 0.8}
|
|
]
|
|
|
|
optimizer._save_results()
|
|
|
|
# write_json_verified should have been called for history and best params
|
|
assert mock_write_json.call_count >= 1
|
|
# joblib.dump should have been called for the study
|
|
assert mock_joblib.dump.called
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.joblib"
|
|
)
|
|
@patch(
|
|
"local_deep_research.security.file_write_verifier.write_json_verified"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_save_results_handles_numpy_types(
|
|
self, mock_evaluator, mock_write_json, mock_joblib
|
|
):
|
|
"""Test that _save_results handles numpy types properly."""
|
|
import tempfile
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
output_dir=tmpdir,
|
|
)
|
|
|
|
mock_study = Mock()
|
|
mock_study.best_params = {"iterations": 2}
|
|
mock_study.best_value = 0.8
|
|
mock_study.best_trial = Mock()
|
|
mock_study.best_trial.user_attrs = {}
|
|
mock_study.trials = []
|
|
optimizer.study = mock_study
|
|
optimizer.best_params = {"iterations": 2}
|
|
optimizer.trials_history = []
|
|
|
|
# Should not raise even with potential numpy types
|
|
optimizer._save_results()
|
|
|
|
|
|
class TestVisualizationCreation:
|
|
"""Tests for visualization creation."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_create_visualizations_handles_no_plotting(self, mock_evaluator):
|
|
"""Test that visualization creation handles missing matplotlib gracefully."""
|
|
import tempfile
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
output_dir=tmpdir,
|
|
)
|
|
|
|
mock_study = Mock()
|
|
mock_study.trials = []
|
|
optimizer.study = mock_study
|
|
optimizer.trials_history = []
|
|
|
|
# Should not raise even if plotting is unavailable
|
|
optimizer._create_visualizations()
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.PLOTTING_AVAILABLE",
|
|
True,
|
|
)
|
|
@patch("local_deep_research.benchmarks.optimization.optuna_optimizer.plt")
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.plot_optimization_history"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.plot_param_importances"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.plot_slice"
|
|
)
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.plot_contour"
|
|
)
|
|
def test_create_visualizations_generates_plots(
|
|
self,
|
|
mock_contour,
|
|
mock_slice,
|
|
mock_importances,
|
|
mock_history,
|
|
mock_plt,
|
|
mock_evaluator,
|
|
):
|
|
"""Test that visualizations are generated when matplotlib is available."""
|
|
import tempfile
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
output_dir=tmpdir,
|
|
)
|
|
|
|
# Need at least 2 trials for visualizations to proceed
|
|
mock_study = Mock()
|
|
mock_study.trials = [Mock(), Mock()]
|
|
mock_study.best_params = {"iterations": 2}
|
|
optimizer.study = mock_study
|
|
optimizer.trials_history = [
|
|
{
|
|
"params": {"iterations": 2},
|
|
"score": 0.8,
|
|
"result": {
|
|
"success": True,
|
|
"quality_score": 0.85,
|
|
"speed_score": 0.75,
|
|
},
|
|
},
|
|
{
|
|
"params": {"iterations": 3},
|
|
"score": 0.7,
|
|
"result": {
|
|
"success": True,
|
|
"quality_score": 0.75,
|
|
"speed_score": 0.65,
|
|
},
|
|
},
|
|
]
|
|
|
|
optimizer._create_visualizations()
|
|
|
|
# Optuna plot functions should have been called
|
|
assert mock_history.called
|
|
|
|
|
|
class TestConvenienceFunctionImplementation:
|
|
"""Tests for convenience function implementation details."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.OptunaOptimizer"
|
|
)
|
|
def test_optimize_for_speed_uses_speed_weights(self, mock_optimizer_class):
|
|
"""Test that optimize_for_speed uses speed-focused weights."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_speed,
|
|
)
|
|
|
|
mock_optimizer = Mock()
|
|
mock_optimizer.optimize.return_value = ({}, 0.0)
|
|
mock_optimizer_class.return_value = mock_optimizer
|
|
|
|
optimize_for_speed(query="test", n_trials=1)
|
|
|
|
# Check that metric_weights have higher speed weight
|
|
call_kwargs = mock_optimizer_class.call_args[1]
|
|
assert (
|
|
call_kwargs["metric_weights"]["speed"]
|
|
> call_kwargs["metric_weights"]["quality"]
|
|
)
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.OptunaOptimizer"
|
|
)
|
|
def test_optimize_for_quality_uses_quality_weights(
|
|
self, mock_optimizer_class
|
|
):
|
|
"""Test that optimize_for_quality uses quality-focused weights."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_quality,
|
|
)
|
|
|
|
mock_optimizer = Mock()
|
|
mock_optimizer.optimize.return_value = ({}, 0.0)
|
|
mock_optimizer_class.return_value = mock_optimizer
|
|
|
|
optimize_for_quality(query="test", n_trials=1)
|
|
|
|
# Check that metric_weights have higher quality weight
|
|
call_kwargs = mock_optimizer_class.call_args[1]
|
|
assert (
|
|
call_kwargs["metric_weights"]["quality"]
|
|
> call_kwargs["metric_weights"]["speed"]
|
|
)
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.OptunaOptimizer"
|
|
)
|
|
def test_optimize_for_efficiency_uses_balanced_weights(
|
|
self, mock_optimizer_class
|
|
):
|
|
"""Test that optimize_for_efficiency uses balanced weights."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
optimize_for_efficiency,
|
|
)
|
|
|
|
mock_optimizer = Mock()
|
|
mock_optimizer.optimize.return_value = ({}, 0.0)
|
|
mock_optimizer_class.return_value = mock_optimizer
|
|
|
|
optimize_for_efficiency(query="test", n_trials=1)
|
|
|
|
# Check that metric_weights include resource
|
|
call_kwargs = mock_optimizer_class.call_args[1]
|
|
assert "resource" in call_kwargs["metric_weights"]
|
|
|
|
|
|
class TestProgressCallback:
|
|
"""Tests for progress callback functionality."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_progress_callback_invoked(self, mock_evaluator):
|
|
"""Test that progress callback is invoked during optimization."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
callback_calls = []
|
|
|
|
def progress_callback(trial_num, n_trials, best_value, best_params):
|
|
callback_calls.append(
|
|
{
|
|
"trial_num": trial_num,
|
|
"n_trials": n_trials,
|
|
"best_value": best_value,
|
|
}
|
|
)
|
|
|
|
optimizer = OptunaOptimizer(
|
|
base_query="test",
|
|
progress_callback=progress_callback,
|
|
)
|
|
|
|
# The callback should be stored
|
|
assert optimizer.progress_callback is progress_callback
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_optimization_callback_method_exists(self, mock_evaluator):
|
|
"""Test that _optimization_callback method exists."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
assert hasattr(optimizer, "_optimization_callback")
|
|
assert callable(optimizer._optimization_callback)
|
|
|
|
|
|
class TestCustomParameterSpace:
|
|
"""Tests for custom parameter space handling."""
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_custom_param_space_used(self, mock_evaluator):
|
|
"""Test that optimize() accepts a custom parameter space."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
import inspect
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
# Verify optimize() accepts param_space parameter
|
|
sig = inspect.signature(optimizer.optimize)
|
|
assert "param_space" in sig.parameters
|
|
|
|
# Verify _get_default_param_space returns a dict with expected keys
|
|
default_space = optimizer._get_default_param_space()
|
|
assert isinstance(default_space, dict)
|
|
assert "iterations" in default_space
|
|
|
|
@patch(
|
|
"local_deep_research.benchmarks.optimization.optuna_optimizer.CompositeBenchmarkEvaluator"
|
|
)
|
|
def test_default_param_space_used_when_none_provided(self, mock_evaluator):
|
|
"""Test that default parameter space is used when none provided."""
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
mock_evaluator.return_value = Mock()
|
|
|
|
optimizer = OptunaOptimizer(base_query="test")
|
|
|
|
# Should use default space
|
|
default_space = optimizer._get_default_param_space()
|
|
assert "iterations" in default_space
|
|
assert "questions_per_iteration" in default_space
|