wehub-resource-sync
7a0da7932b
Backwards Compatibility / Verify Encryption Constants (push) Waiting to run
Backwards Compatibility / PyPI Version Compatibility (push) Waiting to run
Backwards Compatibility / Database Migration Tests (push) Waiting to run
CodeQL Advanced / Analyze (javascript-typescript) (push) Waiting to run
CodeQL Advanced / Analyze (python) (push) Waiting to run
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Blocked by required conditions
Docker Tests (Consolidated) / detect-changes (push) Waiting to run
Docker Tests (Consolidated) / Build Test Image (push) Waiting to run
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Blocked by required conditions
Docker Tests (Consolidated) / Accessibility Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Unit Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Example Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / Production Image Smoke Test (push) Blocked by required conditions
Docker Tests (Consolidated) / Infrastructure Tests (push) Blocked by required conditions
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Waiting to run
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
677 lines
24 KiB
Python
677 lines
24 KiB
Python
"""
|
|
Tests for benchmarks/cli (benchmark_commands.py)
|
|
|
|
Tests cover:
|
|
- setup_benchmark_parser - argument parsing for benchmark commands
|
|
- run_simpleqa_cli - SimpleQA benchmark execution
|
|
- run_browsecomp_cli - BrowseComp benchmark execution
|
|
- list_benchmarks_cli - listing benchmarks
|
|
- main function behavior
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
from unittest.mock import patch, MagicMock
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_data_directory(tmp_path):
|
|
"""Mock the data directory to use a temporary path."""
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.get_data_directory"
|
|
) as mock:
|
|
mock.return_value = tmp_path
|
|
yield tmp_path
|
|
|
|
|
|
class TestSetupBenchmarkParser:
|
|
"""Tests for setup_benchmark_parser function."""
|
|
|
|
def test_simpleqa_command_exists(self, mock_data_directory):
|
|
"""Test that simpleqa command is added to parser."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.command == "simpleqa"
|
|
|
|
def test_browsecomp_command_exists(self, mock_data_directory):
|
|
"""Test that browsecomp command is added to parser."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["browsecomp"])
|
|
assert args.command == "browsecomp"
|
|
|
|
def test_list_command_exists(self, mock_data_directory):
|
|
"""Test that list command is added to parser."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["list"])
|
|
assert args.command == "list"
|
|
|
|
def test_compare_command_exists(self, mock_data_directory):
|
|
"""Test that compare command is added to parser."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["compare"])
|
|
assert args.command == "compare"
|
|
|
|
def test_simpleqa_default_examples(self, mock_data_directory):
|
|
"""Test that simpleqa has default examples of 100."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.examples == 100
|
|
|
|
def test_simpleqa_custom_examples(self, mock_data_directory):
|
|
"""Test that simpleqa accepts custom examples."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--examples", "50"])
|
|
assert args.examples == 50
|
|
|
|
def test_simpleqa_default_iterations(self, mock_data_directory):
|
|
"""Test that simpleqa has default iterations of 3."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.iterations == 3
|
|
|
|
def test_simpleqa_custom_iterations(self, mock_data_directory):
|
|
"""Test that simpleqa accepts custom iterations."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--iterations", "5"])
|
|
assert args.iterations == 5
|
|
|
|
def test_simpleqa_default_questions(self, mock_data_directory):
|
|
"""Test that simpleqa has default questions of 3."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.questions == 3
|
|
|
|
def test_simpleqa_default_search_tool(self, mock_data_directory):
|
|
"""Test that simpleqa has default search_tool of searxng."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.search_tool == "searxng"
|
|
|
|
def test_simpleqa_custom_search_tool(self, mock_data_directory):
|
|
"""Test that simpleqa accepts custom search_tool."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--search-tool", "duckduckgo"])
|
|
assert args.search_tool == "duckduckgo"
|
|
|
|
def test_simpleqa_human_eval_flag(self, mock_data_directory):
|
|
"""Test that simpleqa accepts human-eval flag."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--human-eval"])
|
|
assert args.human_eval is True
|
|
|
|
def test_simpleqa_no_eval_flag(self, mock_data_directory):
|
|
"""Test that simpleqa accepts no-eval flag."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--no-eval"])
|
|
assert args.no_eval is True
|
|
|
|
def test_simpleqa_custom_output_dir(self, mock_data_directory):
|
|
"""Test that simpleqa accepts custom output-dir."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--output-dir", "/custom/path"])
|
|
assert args.output_dir == "/custom/path"
|
|
|
|
def test_simpleqa_search_model_option(self, mock_data_directory):
|
|
"""Test that simpleqa accepts search-model option."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--search-model", "gpt-4"])
|
|
assert args.search_model == "gpt-4"
|
|
|
|
def test_simpleqa_search_provider_option(self, mock_data_directory):
|
|
"""Test that simpleqa accepts search-provider option."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--search-provider", "openai"])
|
|
assert args.search_provider == "openai"
|
|
|
|
def test_simpleqa_search_strategy_option(self, mock_data_directory):
|
|
"""Test that simpleqa accepts search-strategy option."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa", "--search-strategy", "parallel"])
|
|
assert args.search_strategy == "parallel"
|
|
|
|
def test_simpleqa_default_search_strategy(self, mock_data_directory):
|
|
"""Test that simpleqa has default search-strategy of source_based."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["simpleqa"])
|
|
assert args.search_strategy == "source_based"
|
|
|
|
def test_compare_default_dataset(self, mock_data_directory):
|
|
"""Test that compare has default dataset of simpleqa."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["compare"])
|
|
assert args.dataset == "simpleqa"
|
|
|
|
def test_compare_custom_dataset(self, mock_data_directory):
|
|
"""Test that compare accepts custom dataset."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["compare", "--dataset", "browsecomp"])
|
|
assert args.dataset == "browsecomp"
|
|
|
|
def test_compare_default_examples(self, mock_data_directory):
|
|
"""Test that compare has default examples of 20."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
setup_benchmark_parser,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
setup_benchmark_parser(subparsers)
|
|
|
|
args = parser.parse_args(["compare"])
|
|
assert args.examples == 20
|
|
|
|
|
|
class TestRunSimpleqaCli:
|
|
"""Tests for run_simpleqa_cli function."""
|
|
|
|
def test_run_simpleqa_calls_benchmark(self, mock_data_directory):
|
|
"""Test that run_simpleqa_cli calls run_simpleqa_benchmark."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_simpleqa_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 2
|
|
args.questions = 2
|
|
args.search_tool = "searxng"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = None
|
|
args.eval_provider = None
|
|
args.search_model = None
|
|
args.search_provider = None
|
|
args.endpoint_url = None
|
|
args.search_strategy = "source_based"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {
|
|
"metrics": {
|
|
"accuracy": 0.8,
|
|
"correct": 8,
|
|
"average_processing_time": 5.0,
|
|
},
|
|
"total_examples": 10,
|
|
"report_path": "/tmp/report.html",
|
|
}
|
|
|
|
run_simpleqa_cli(args)
|
|
|
|
mock_benchmark.assert_called_once()
|
|
|
|
def test_run_simpleqa_passes_search_config(self, mock_data_directory):
|
|
"""Test that run_simpleqa_cli passes search config correctly."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_simpleqa_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 5
|
|
args.questions = 4
|
|
args.search_tool = "duckduckgo"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = None
|
|
args.eval_provider = None
|
|
args.search_model = "gpt-4"
|
|
args.search_provider = "openai"
|
|
args.endpoint_url = None
|
|
args.search_strategy = "parallel"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {"metrics": {}, "total_examples": 10}
|
|
|
|
run_simpleqa_cli(args)
|
|
|
|
call_kwargs = mock_benchmark.call_args[1]
|
|
assert call_kwargs["search_config"]["iterations"] == 5
|
|
assert call_kwargs["search_config"]["questions_per_iteration"] == 4
|
|
assert call_kwargs["search_config"]["search_tool"] == "duckduckgo"
|
|
assert call_kwargs["search_config"]["model_name"] == "gpt-4"
|
|
assert call_kwargs["search_config"]["provider"] == "openai"
|
|
|
|
|
|
class TestRunBrowsecompCli:
|
|
"""Tests for run_browsecomp_cli function."""
|
|
|
|
def test_run_browsecomp_calls_benchmark(self, mock_data_directory):
|
|
"""Test that run_browsecomp_cli calls run_browsecomp_benchmark."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_browsecomp_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 2
|
|
args.questions = 2
|
|
args.search_tool = "searxng"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = None
|
|
args.eval_provider = None
|
|
args.search_model = None
|
|
args.search_provider = None
|
|
args.endpoint_url = None
|
|
args.search_strategy = "source_based"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_browsecomp_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {
|
|
"metrics": {
|
|
"accuracy": 0.7,
|
|
"correct": 7,
|
|
"average_processing_time": 6.0,
|
|
},
|
|
"total_examples": 10,
|
|
"report_path": "/tmp/report.html",
|
|
}
|
|
|
|
run_browsecomp_cli(args)
|
|
|
|
mock_benchmark.assert_called_once()
|
|
|
|
|
|
class TestListBenchmarksCli:
|
|
"""Tests for list_benchmarks_cli function."""
|
|
|
|
def test_list_benchmarks_calls_get_available_datasets(
|
|
self, mock_data_directory, capsys
|
|
):
|
|
"""Test that list_benchmarks_cli calls get_available_datasets."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
list_benchmarks_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
|
|
) as mock_datasets:
|
|
mock_datasets.return_value = [
|
|
{
|
|
"id": "simpleqa",
|
|
"name": "SimpleQA",
|
|
"description": "Simple QA benchmark",
|
|
"url": "http://example.com",
|
|
}
|
|
]
|
|
|
|
list_benchmarks_cli(args)
|
|
|
|
mock_datasets.assert_called_once()
|
|
|
|
def test_list_benchmarks_prints_datasets(self, mock_data_directory, capsys):
|
|
"""Test that list_benchmarks_cli prints dataset information."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
list_benchmarks_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
|
|
) as mock_datasets:
|
|
mock_datasets.return_value = [
|
|
{
|
|
"id": "simpleqa",
|
|
"name": "SimpleQA",
|
|
"description": "Simple QA benchmark",
|
|
"url": "http://example.com",
|
|
}
|
|
]
|
|
|
|
list_benchmarks_cli(args)
|
|
|
|
captured = capsys.readouterr()
|
|
assert "simpleqa" in captured.out
|
|
assert "SimpleQA" in captured.out
|
|
|
|
|
|
class TestMain:
|
|
"""Tests for main function."""
|
|
|
|
def test_main_requires_command(self, mock_data_directory):
|
|
"""Test that main requires a command."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import main
|
|
|
|
with patch.object(sys, "argv", ["ldr-benchmark"]):
|
|
with pytest.raises(SystemExit):
|
|
main()
|
|
|
|
def test_main_with_list_command(self, mock_data_directory, capsys):
|
|
"""Test that main handles list command."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import main
|
|
|
|
with patch.object(sys, "argv", ["ldr-benchmark", "list"]):
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets"
|
|
) as mock_datasets:
|
|
mock_datasets.return_value = [
|
|
{
|
|
"id": "test",
|
|
"name": "Test",
|
|
"description": "Test",
|
|
"url": "http://test.com",
|
|
}
|
|
]
|
|
main()
|
|
|
|
captured = capsys.readouterr()
|
|
assert "Available Benchmarks" in captured.out
|
|
|
|
def test_main_pins_diagnose_false_on_logger_add(self, mock_data_directory):
|
|
"""main() must pass diagnose=False to every logger.add (issue #4185).
|
|
|
|
loguru defaults diagnose=True, which renders repr() of every
|
|
traceback frame's locals on exception. Benchmarks invoke LLMs and
|
|
search engines whose handlers hold api_key/Authorization headers as
|
|
frame locals; leaving the default on would dump those to stderr on
|
|
any failure. The benchmark CLI bypasses config_logger, so the fix
|
|
lives at this entry point.
|
|
"""
|
|
import local_deep_research.benchmarks.cli.benchmark_commands as module
|
|
|
|
for verbose_flag in (["--verbose"], []):
|
|
argv = ["ldr-benchmark"] + verbose_flag + ["list"]
|
|
with patch.object(sys, "argv", argv):
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.get_available_datasets",
|
|
return_value=[],
|
|
):
|
|
with patch.object(module, "logger") as mock_logger:
|
|
module.main()
|
|
|
|
assert mock_logger.add.call_args_list, (
|
|
f"no logger.add calls observed for argv={argv}"
|
|
)
|
|
for add_call in mock_logger.add.call_args_list:
|
|
assert add_call.kwargs.get("diagnose") is False, (
|
|
f"benchmark CLI logger.add must pass "
|
|
f"diagnose=False; got "
|
|
f"{add_call.kwargs.get('diagnose')!r} "
|
|
f"for argv={argv}"
|
|
)
|
|
|
|
|
|
class TestSearchConfigBuilding:
|
|
"""Tests for search config building logic."""
|
|
|
|
def test_search_config_includes_basic_params(self, mock_data_directory):
|
|
"""Test that search config includes basic parameters."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_simpleqa_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 3
|
|
args.questions = 2
|
|
args.search_tool = "searxng"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = None
|
|
args.eval_provider = None
|
|
args.search_model = None
|
|
args.search_provider = None
|
|
args.endpoint_url = None
|
|
args.search_strategy = "standard"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {"metrics": {}, "total_examples": 10}
|
|
|
|
run_simpleqa_cli(args)
|
|
|
|
call_kwargs = mock_benchmark.call_args[1]
|
|
assert "search_config" in call_kwargs
|
|
assert call_kwargs["search_config"]["iterations"] == 3
|
|
assert call_kwargs["search_config"]["questions_per_iteration"] == 2
|
|
assert call_kwargs["search_config"]["search_tool"] == "searxng"
|
|
|
|
def test_evaluation_config_set_when_eval_model_provided(
|
|
self, mock_data_directory
|
|
):
|
|
"""Test that evaluation config is set when eval_model is provided."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_simpleqa_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 3
|
|
args.questions = 2
|
|
args.search_tool = "searxng"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = "gpt-4"
|
|
args.eval_provider = "openai"
|
|
args.search_model = None
|
|
args.search_provider = None
|
|
args.endpoint_url = None
|
|
args.search_strategy = "standard"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {"metrics": {}, "total_examples": 10}
|
|
|
|
run_simpleqa_cli(args)
|
|
|
|
call_kwargs = mock_benchmark.call_args[1]
|
|
assert call_kwargs["evaluation_config"] is not None
|
|
assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4"
|
|
assert call_kwargs["evaluation_config"]["provider"] == "openai"
|
|
|
|
def test_evaluation_config_none_when_no_eval_args(
|
|
self, mock_data_directory
|
|
):
|
|
"""Test that evaluation config is None when no eval args provided."""
|
|
from local_deep_research.benchmarks.cli.benchmark_commands import (
|
|
run_simpleqa_cli,
|
|
)
|
|
|
|
args = MagicMock()
|
|
args.examples = 10
|
|
args.iterations = 3
|
|
args.questions = 2
|
|
args.search_tool = "searxng"
|
|
args.output_dir = "/tmp/output"
|
|
args.human_eval = False
|
|
args.no_eval = False
|
|
args.custom_dataset = None
|
|
args.eval_model = None
|
|
args.eval_provider = None
|
|
args.search_model = None
|
|
args.search_provider = None
|
|
args.endpoint_url = None
|
|
args.search_strategy = "standard"
|
|
|
|
with patch(
|
|
"local_deep_research.benchmarks.cli.benchmark_commands.run_simpleqa_benchmark"
|
|
) as mock_benchmark:
|
|
mock_benchmark.return_value = {"metrics": {}, "total_examples": 10}
|
|
|
|
run_simpleqa_cli(args)
|
|
|
|
call_kwargs = mock_benchmark.call_args[1]
|
|
assert call_kwargs["evaluation_config"] is None
|