chore: import upstream snapshot with attribution

examples/optimization/run_parallel_benchmark.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python
+"""
+Run SimpleQA and BrowseComp benchmarks in parallel with 300 examples each.
+
+This script demonstrates running multiple benchmarks in parallel with a large number of examples.
+
+Usage:
+    # Install dependencies with PDM
+    cd /path/to/local-deep-research
+    pdm install
+
+    # Run the script with PDM
+    pdm run python examples/optimization/run_parallel_benchmark.py
+"""
+
+import argparse
+import concurrent.futures
+import os
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+
+from loguru import logger
+
+# Add the src directory to the Python path
+project_root = str(Path(__file__).parent.parent.parent.resolve())
+sys.path.insert(0, str(Path(project_root) / "src"))
+
+
+def run_simpleqa_benchmark(
+    num_examples,
+    output_dir,
+    model=None,
+    provider=None,
+    endpoint_url=None,
+    api_key=None,
+):
+    """Run SimpleQA benchmark with specified number of examples."""
+    from local_deep_research.benchmarks.benchmark_functions import (
+        evaluate_simpleqa,
+    )
+
+    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
+    start_time = time.time()
+
+    # Run the benchmark
+    results = evaluate_simpleqa(
+        num_examples=num_examples,
+        search_iterations=2,
+        questions_per_iteration=3,
+        search_strategy="source_based",
+        search_tool="searxng",
+        search_model=model,
+        search_provider=provider,
+        endpoint_url=endpoint_url,
+        output_dir=str(Path(output_dir) / "simpleqa"),
+        evaluation_provider="ANTHROPIC",
+        evaluation_model="claude-3-7-sonnet-20250219",
+    )
+
+    duration = time.time() - start_time
+    logger.info(f"SimpleQA benchmark completed in {duration:.1f} seconds")
+
+    if results and isinstance(results, dict):
+        logger.info(f"SimpleQA accuracy: {results.get('accuracy', 'N/A')}")
+
+    return results
+
+
+def run_browsecomp_benchmark(
+    num_examples,
+    output_dir,
+    model=None,
+    provider=None,
+    endpoint_url=None,
+    api_key=None,
+):
+    """Run BrowseComp benchmark with specified number of examples."""
+    from local_deep_research.benchmarks.benchmark_functions import (
+        evaluate_browsecomp,
+    )
+
+    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
+    start_time = time.time()
+
+    # Run the benchmark
+    results = evaluate_browsecomp(
+        num_examples=num_examples,
+        search_iterations=3,
+        questions_per_iteration=3,
+        search_strategy="source_based",
+        search_tool="searxng",
+        search_model=model,
+        search_provider=provider,
+        endpoint_url=endpoint_url,
+        output_dir=str(Path(output_dir) / "browsecomp"),
+        evaluation_provider="ANTHROPIC",
+        evaluation_model="claude-3-7-sonnet-20250219",
+    )
+
+    duration = time.time() - start_time
+    logger.info(f"BrowseComp benchmark completed in {duration:.1f} seconds")
+
+    if results and isinstance(results, dict):
+        logger.info(f"BrowseComp accuracy: {results.get('accuracy', 'N/A')}")
+
+    return results
+
+
+def setup_llm_environment(
+    model=None, provider=None, endpoint_url=None, api_key=None
+):
+    """Set up environment variables for LLM configuration."""
+    if model:
+        os.environ["LDR_LLM__MODEL"] = model
+        logger.info(f"Using LLM model: {model}")
+
+    if provider:
+        os.environ["LDR_LLM__PROVIDER"] = provider
+        logger.info(f"Using LLM provider: {provider}")
+
+    if endpoint_url:
+        os.environ["OPENAI_ENDPOINT_URL"] = endpoint_url
+        os.environ["LDR_LLM__OPENAI_ENDPOINT_URL"] = endpoint_url
+        logger.info(f"Using endpoint URL: {endpoint_url}")
+
+    if api_key:
+        # Set the appropriate environment variable based on provider
+        if provider == "openai":
+            os.environ["OPENAI_API_KEY"] = api_key
+            os.environ["LDR_LLM__OPENAI_API_KEY"] = api_key
+        elif provider == "openai_endpoint":
+            os.environ["OPENAI_ENDPOINT_API_KEY"] = api_key
+            os.environ["LDR_LLM__OPENAI_ENDPOINT_API_KEY"] = api_key
+        elif provider == "anthropic":
+            os.environ["ANTHROPIC_API_KEY"] = api_key
+            os.environ["LDR_LLM__ANTHROPIC_API_KEY"] = api_key
+
+        logger.info("API key configured")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Run SimpleQA and BrowseComp benchmarks in parallel"
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=300,
+        help="Number of examples for each benchmark (default: 300)",
+    )
+
+    # LLM configuration options
+    parser.add_argument(
+        "--model",
+        help="Model name for the LLM (e.g., 'claude-3-sonnet-20240229')",
+    )
+    parser.add_argument(
+        "--provider",
+        help="Provider for the LLM (e.g., 'anthropic', 'openai', 'openai_endpoint')",
+    )
+    parser.add_argument(
+        "--endpoint-url",
+        help="Custom endpoint URL (e.g., 'https://openrouter.ai/api/v1')",
+    )
+    parser.add_argument("--api-key", help="API key for the LLM provider")
+
+    args = parser.parse_args()
+
+    # Create timestamp for unique output directory
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    output_dir = str(
+        Path(project_root)
+        / "benchmark_results"
+        / f"parallel_benchmark_{timestamp}"
+    )
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+    # Display start information
+    print(f"Starting parallel benchmarks with {args.examples} examples each")
+    print(f"Results will be saved to: {output_dir}")
+
+    # Set up LLM environment if specified
+    setup_llm_environment(
+        model=args.model,
+        provider=args.provider,
+        endpoint_url=args.endpoint_url,
+        api_key=args.api_key,
+    )
+
+    # Start time for total execution
+    total_start_time = time.time()
+
+    # Run benchmarks in parallel using ThreadPoolExecutor
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        # Submit both benchmark jobs
+        simpleqa_future = executor.submit(
+            run_simpleqa_benchmark,
+            args.examples,
+            output_dir,
+            args.model,
+            args.provider,
+            args.endpoint_url,
+            args.api_key,
+        )
+
+        browsecomp_future = executor.submit(
+            run_browsecomp_benchmark,
+            args.examples,
+            output_dir,
+            args.model,
+            args.provider,
+            args.endpoint_url,
+            args.api_key,
+        )
+
+        # Get results from both futures
+        try:
+            simpleqa_results = simpleqa_future.result()
+            print("SimpleQA benchmark completed successfully")
+        except Exception:
+            logger.exception("Error in SimpleQA benchmark")
+            simpleqa_results = None
+
+        try:
+            browsecomp_results = browsecomp_future.result()
+            print("BrowseComp benchmark completed successfully")
+        except Exception:
+            logger.exception("Error in BrowseComp benchmark")
+            browsecomp_results = None
+
+    # Calculate total time
+    total_duration = time.time() - total_start_time
+
+    # Print summary
+    print("\n" + "=" * 50)
+    print(" PARALLEL BENCHMARK SUMMARY ")
+    print("=" * 50)
+    print(f"Total duration: {total_duration:.1f} seconds")
+    print(f"Examples per benchmark: {args.examples}")
+
+    if simpleqa_results and isinstance(simpleqa_results, dict):
+        print(f"SimpleQA accuracy: {simpleqa_results.get('accuracy', 'N/A')}")
+    else:
+        print("SimpleQA: Failed or no results")
+
+    if browsecomp_results and isinstance(browsecomp_results, dict):
+        print(
+            f"BrowseComp accuracy: {browsecomp_results.get('accuracy', 'N/A')}"
+        )
+    else:
+        print("BrowseComp: Failed or no results")
+
+    print(f"Results saved to: {output_dir}")
+    print("=" * 50)
+
+    # Save summary to JSON file
+    try:
+        import json
+
+        summary = {
+            "timestamp": timestamp,
+            "examples_per_benchmark": args.examples,
+            "total_duration": total_duration,
+            "simpleqa": {
+                "accuracy": (
+                    simpleqa_results.get("accuracy")
+                    if simpleqa_results
+                    else None
+                ),
+                "completed": simpleqa_results is not None,
+            },
+            "browsecomp": {
+                "accuracy": (
+                    browsecomp_results.get("accuracy")
+                    if browsecomp_results
+                    else None
+                ),
+                "completed": browsecomp_results is not None,
+            },
+            "model": args.model,
+            "provider": args.provider,
+        }
+
+        with open(
+            Path(output_dir) / "parallel_benchmark_summary.json",
+            "w",
+            encoding="utf-8",
+        ) as f:
+            json.dump(summary, f, indent=2)
+
+    except Exception:
+        logger.exception("Error saving summary")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())