learningcircuit--local-deep-research/examples/benchmarks/gemini/run_gemini_benchmark_fixed.py

#!/usr/bin/env python
"""
Fixed benchmark with Gemini 2.0 Flash via OpenRouter
"""

import sys
import time
from datetime import datetime, UTC
from pathlib import Path

# Import the benchmark functions
from local_deep_research.benchmarks.benchmark_functions import (
    evaluate_browsecomp,
    evaluate_simpleqa,
)

# Monkey patch the get_llm function to use Gemini
from local_deep_research.config import llm_config

# Save original function
original_get_llm = llm_config.get_llm


def setup_gemini_config():
    """
    Create a custom evaluation configuration using Gemini 2.0 Flash via OpenRouter
    """
    # Configure to use Gemini 2.0 Flash via OpenRouter
    evaluation_config = {
        "model_name": "google/gemini-2.0-flash-001",  # OpenRouter format for Gemini
        "provider": "openai_endpoint",  # Use OpenRouter as endpoint
        "openai_endpoint_url": "https://openrouter.ai/api/v1",
        "temperature": 0,  # Zero temp for consistent evaluation
    }

    print(f"Using Gemini 2.0 Flash for evaluation: {evaluation_config}")
    return evaluation_config


# Override get_llm to always use Gemini
def patched_get_llm(
    model_name=None, temperature=None, provider=None, openai_endpoint_url=None
):
    """Patched version that always uses Gemini via OpenRouter"""
    if (
        model_name == "gemma3:12b"
    ):  # This is the default model that causes the error
        print("Overriding local model with Gemini 2.0 Flash")
        model_name = "google/gemini-2.0-flash-001"
        provider = "openai_endpoint"
        openai_endpoint_url = "https://openrouter.ai/api/v1"
    return original_get_llm(
        model_name, temperature, provider, openai_endpoint_url
    )


# Apply the patch
llm_config.get_llm = patched_get_llm


def run_benchmark(examples=1):
    """Run benchmarks with Gemini 2.0 Flash"""
    try:
        # Create timestamp for output
        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
        output_dir = str(
            Path(__file__).parent.parent.parent
            / "benchmark_results"
            / f"gemini_eval_{timestamp}"
        )
        Path(output_dir).mkdir(parents=True, exist_ok=True)

        # Setup the Gemini configuration
        gemini_config = setup_gemini_config()

        # Run SimpleQA benchmark
        print(f"\n=== Running SimpleQA benchmark with {examples} examples ===")
        simpleqa_start = time.time()

        simpleqa_results = evaluate_simpleqa(
            num_examples=examples,
            search_iterations=2,
            questions_per_iteration=3,
            search_tool="searxng",
            evaluation_model=gemini_config["model_name"],
            evaluation_provider=gemini_config["provider"],
            output_dir=str(Path(output_dir) / "simpleqa"),
        )

        simpleqa_duration = time.time() - simpleqa_start
        print(
            f"SimpleQA evaluation complete in {simpleqa_duration:.1f} seconds"
        )
        if (
            isinstance(simpleqa_results, dict)
            and "accuracy" in simpleqa_results
        ):
            print(f"SimpleQA accuracy: {simpleqa_results['accuracy']:.4f}")
        else:
            print("SimpleQA accuracy: N/A")

        # Run BrowseComp benchmark
        print(
            f"\n=== Running BrowseComp benchmark with {examples} examples ==="
        )
        browsecomp_start = time.time()

        browsecomp_results = evaluate_browsecomp(
            num_examples=examples,
            search_iterations=3,
            questions_per_iteration=3,
            search_tool="searxng",
            evaluation_model=gemini_config["model_name"],
            evaluation_provider=gemini_config["provider"],
            output_dir=str(Path(output_dir) / "browsecomp"),
        )

        browsecomp_duration = time.time() - browsecomp_start
        print(
            f"BrowseComp evaluation complete in {browsecomp_duration:.1f} seconds"
        )
        if (
            isinstance(browsecomp_results, dict)
            and "accuracy" in browsecomp_results
        ):
            print(f"BrowseComp accuracy: {browsecomp_results['accuracy']:.4f}")
        else:
            print("BrowseComp accuracy: N/A")

        # Generate summary
        print("\n=== Evaluation Summary ===")
        print(f"Examples: {examples}")
        print(f"Model: {gemini_config.get('model_name', 'unknown')}")
        print(f"Provider: {gemini_config.get('provider', 'unknown')}")
        print(f"Results saved to: {output_dir}")

        return {
            "simpleqa": simpleqa_results,
            "browsecomp": browsecomp_results,
        }

    except Exception as e:
        print(f"Error running benchmark: {e}")
        import traceback

        traceback.print_exc()
        return None


def main():
    # Parse command line arguments
    import argparse

    parser = argparse.ArgumentParser(
        description="Run benchmark with Gemini 2.0 Flash"
    )
    parser.add_argument(
        "--examples",
        type=int,
        default=1,
        help="Number of examples to evaluate (default: 1)",
    )

    args = parser.parse_args()

    print(
        f"Starting benchmark with Gemini 2.0 Flash on {args.examples} examples"
    )

    # Run the evaluation
    results = run_benchmark(examples=args.examples)

    # Return success if benchmark completed
    return 0 if results else 1


if __name__ == "__main__":
    sys.exit(main())