learningcircuit--local-deep-research/examples/optimization/browsecomp_optimization.py

#!/usr/bin/env python
"""
Parameter Optimization Using BrowseComp Benchmark for Local Deep Research.

This script demonstrates optimizing research parameters using the BrowseComp benchmark
for higher quality evaluation.

Usage:
    # Install dependencies with PDM
    cd /path/to/local-deep-research
    pdm install

    # Run the script with PDM
    pdm run python examples/optimization/browsecomp_optimization.py
"""

import json
import sys
from datetime import datetime
from pathlib import Path


from local_deep_research.benchmarks.optimization import optimize_parameters

# Add the src directory to the Python path
project_root = str(Path(__file__).parent.parent.parent.resolve())
sys.path.insert(0, str(Path(project_root) / "src"))


def main():
    # Create timestamp for unique output directory
    from datetime import timezone

    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
    output_dir = str(
        Path("examples")
        / "optimization"
        / "results"
        / f"browsecomp_opt_{timestamp}"
    )
    Path(output_dir).mkdir(parents=True, exist_ok=True)

    print(
        f"Starting BrowseComp optimization - results will be saved to {output_dir}"
    )

    # Define a simple parameter space for demonstration
    param_space = {
        "iterations": {
            "type": "int",
            "low": 1,
            "high": 3,
            "step": 1,
        },
        "questions_per_iteration": {
            "type": "int",
            "low": 1,
            "high": 3,
            "step": 1,
        },
        "search_strategy": {
            "type": "categorical",
            "choices": ["rapid", "standard", "parallel"],
        },
    }

    # Run optimization with BrowseComp benchmark
    # Using a small number of trials and examples for demonstration
    print("\n=== Running balanced optimization with BrowseComp benchmark ===")
    balanced_params, balanced_score = optimize_parameters(
        query="Climate change effects on biodiversity",
        param_space=param_space,
        output_dir=output_dir,
        n_trials=3,  # Small number for demo purposes
        search_tool="searxng",
        benchmark_weights={
            "browsecomp": 1.0
        },  # Specify BrowseComp benchmark only
    )

    print(f"Best balanced parameters: {balanced_params}")
    print(f"Best balanced score: {balanced_score:.4f}")

    # Save optimization results
    summary = {
        "timestamp": timestamp,
        "benchmark_weights": {"browsecomp": 1.0},
        "balanced": {
            "parameters": balanced_params,
            "score": float(balanced_score),
        },
    }

    with open(
        Path(output_dir) / "browsecomp_optimization_summary.json",
        "w",
        encoding="utf-8",
    ) as f:
        json.dump(summary, f, indent=2)

    print(
        f"\nDemo complete! Results saved to {output_dir}/browsecomp_optimization_summary.json"
    )
    print(f"Recommended parameters for BrowseComp: {balanced_params}")

    print(
        "\nNote: For actual optimizations, we recommend increasing n_trials to at least 20."
    )
    print(
        "This demo runs with minimal trials to demonstrate the functionality quickly."
    )


if __name__ == "__main__":
    main()