chore: import upstream snapshot with attribution

examples/benchmarks/README.md

@@ -0,0 +1,62 @@
+# Benchmarks for Local Deep Research
+
+This directory contains scripts for running benchmarks to evaluate Local Deep Research's performance.
+
+## Available Benchmarks
+
+### SimpleQA
+
+The SimpleQA benchmark evaluates factual question answering capabilities.
+
+```bash
+python run_simpleqa.py --examples 10 --iterations 3 --questions 3
+```
+
+Options:
+- `--examples`: Number of examples to run (default: 10)
+- `--iterations`: Number of search iterations (default: 3)
+- `--questions`: Questions per iteration (default: 3)
+- `--search-tool`: Search tool to use (default: "searxng")
+- `--output-dir`: Directory to save results (default: "benchmark_results")
+- `--no-eval`: Skip evaluation
+- `--human-eval`: Use human evaluation
+- `--eval-model`: Model to use for evaluation
+- `--eval-provider`: Provider to use for evaluation
+
+### BrowseComp
+
+The BrowseComp benchmark evaluates web browsing comprehension and complex question answering.
+
+```bash
+python run_browsecomp.py --examples 5 --iterations 3 --questions 3
+```
+
+Options:
+- `--examples`: Number of examples to run (default: 2)
+- `--iterations`: Number of search iterations (default: 1)
+- `--questions`: Questions per iteration (default: 1)
+- `--search-tool`: Search tool to use (default: "searxng")
+- `--output-dir`: Directory to save results (default: "browsecomp_results")
+
+See `browsecomp_benchmark_readme.md` for more information on how BrowseComp works.
+
+## Running All Benchmarks
+
+To run both benchmarks and compare results:
+
+```bash
+# Run SimpleQA with default settings
+python run_simpleqa.py
+
+# Run BrowseComp with increased iterations and questions
+python run_browsecomp.py --iterations 3 --questions 3
+```
+
+## Evaluating Results
+
+Results are saved in the specified output directories and include:
+- Raw results (JSONL format)
+- Evaluation results (JSONL format)
+- Summary reports (Markdown format)
+
+The scripts will also print a summary of the results to the console, including accuracy metrics.

examples/benchmarks/browsecomp/run_browsecomp_fixed_v2.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+"""
+Fixed BrowseComp benchmark script that properly handles decryption.
+This version uses the canary string from each example as the decryption key.
+"""
+
+import base64
+import hashlib
+import json
+import os
+import random
+import re
+import sys
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+
+# Set up Python path
+current_dir = str(Path(__file__).parent.resolve())
+sys.path.insert(0, current_dir)
+
+try:
+    from local_deep_research.api import quick_summary
+except ImportError as e:
+    print(f"Error importing from api: {e}")
+    sys.exit(1)
+
+# The template for formatting queries to the model
+QUERY_TEMPLATE = """
+{Question}
+
+Your response should be in the following format:
+Explanation: {{your explanation for your final answer}}
+Exact Answer: {{your succinct, final answer}}
+Confidence: {{your confidence score between 0% and 100% for your answer}}
+""".strip()
+
+# Grader template from BrowseComp
+GRADER_TEMPLATE = """
+Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+Your judgement must be in the format and criteria specified below:
+
+extracted_final_answer: The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response.
+
+[correct_answer]: {correct_answer}
+
+reasoning: Explain why the extracted_final_answer is correct or incorrect based on [correct_answer], focusing only on if there are meaningful differences between [correct_answer] and the extracted_final_answer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match.
+
+correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect.
+
+confidence: The extracted confidence score between 0% and 100% from [response]. Put 100 if there is no confidence score available.
+""".strip()
+
+
+def derive_key(password: str, length: int) -> bytes:
+    """Derive a fixed-length key from the password using SHA256."""
+    hasher = hashlib.sha256()
+    hasher.update(password.encode())
+    key = hasher.digest()
+    return key * (length // len(key)) + key[: length % len(key)]
+
+
+def decrypt(ciphertext_b64: str, password: str) -> str:
+    """Decrypt base64-encoded ciphertext with XOR."""
+    try:
+        encrypted = base64.b64decode(ciphertext_b64)
+        key = derive_key(password, len(encrypted))
+        decrypted = bytes(a ^ b for a, b in zip(encrypted, key, strict=False))
+        return decrypted.decode()
+    except Exception as e:
+        print(f"Error decrypting data: {e!s}")
+        return f"Error: Could not decrypt data: {str(e)[:100]}"
+
+
+def run_browsecomp_evaluation(
+    dataset_path: str = "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv",
+    output_dir: str = "benchmark_results/browsecomp",
+    output_file: str = "ldr_browsecomp_results.jsonl",
+    num_examples: Optional[int] = None,
+    seed: int = 42,
+    search_iterations: int = 2,
+    questions_per_iteration: int = 9,
+    search_tool: str = "searxng",
+):
+    """
+    Run the BrowseComp evaluation using Local Deep Research.
+    """
+    # Ensure output directory exists
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    output_path = str(Path(output_dir) / output_file)
+
+    # Load BrowseComp dataset
+    print(f"Loading dataset from {dataset_path}")
+    df = pd.read_csv(dataset_path)
+
+    examples = [row.to_dict() for _, row in df.iterrows()]
+
+    # Display sample canary
+    if examples:
+        print(
+            f"Sample canary: {examples[0].get('canary', 'No canary found')[:30]}..."
+        )
+
+    # Sample examples if specified
+    if num_examples and num_examples < len(examples):
+        random.seed(seed)
+        examples = random.sample(examples, num_examples)
+        print(f"Sampled {num_examples} examples from {len(df)} total examples")
+
+    # Remove output file if it exists to avoid appending
+    if Path(output_path).exists():
+        os.remove(output_path)
+
+    results = []
+    correct_count = 0
+
+    print("\nStarting BrowseComp evaluation with settings:")
+    print(f"- Number of examples: {len(examples)}")
+    print(f"- Search iterations: {search_iterations}")
+    print(f"- Questions per iteration: {questions_per_iteration}")
+    print(f"- Search tool: {search_tool}")
+    print(f"- Output file: {output_path}")
+
+    # Process each question
+    for i, example in enumerate(examples):
+        # Decrypt the problem and answer using the canary
+        try:
+            problem = decrypt(
+                example.get("problem", ""), example.get("canary", "")
+            )
+            correct_answer = decrypt(
+                example.get("answer", ""), example.get("canary", "")
+            )
+
+            print(f"\nProcessing {i + 1}/{len(examples)}: {problem[:100]}...")
+            print(f"Correct answer: {correct_answer[:100]}...")
+        except Exception as e:
+            print(f"Error decrypting problem/answer: {e}")
+            problem = f"Error decrypting: {str(e)[:50]}"
+            correct_answer = "Unknown due to decryption error"
+
+        # Format the question using the QUERY_TEMPLATE
+        formatted_question = QUERY_TEMPLATE.format(Question=problem)
+
+        try:
+            # Query using quick_summary with specified parameters
+            summary = quick_summary(
+                query=formatted_question,
+                iterations=search_iterations,
+                questions_per_iteration=questions_per_iteration,
+                search_tool=search_tool,
+            )
+
+            # Extract the response
+            response = summary.get("summary", "")
+
+            # Clean up the response for better evaluation
+            response = (
+                response.replace("[1]", "")
+                .replace("[2]", "")
+                .replace("[3]", "")
+            )
+            response = " ".join(
+                [
+                    line
+                    for line in response.split("\n")
+                    if not line.startswith("[")
+                ]
+            )
+
+            # Extract the final answer from the response
+            answer_match = re.search(r"Exact Answer:\s*(.*?)(?:\n|$)", response)
+            exact_answer = (
+                answer_match.group(1).strip() if answer_match else "None"
+            )
+
+            # Extract confidence from the response
+            confidence_match = re.search(r"Confidence:\s*(\d+)%", response)
+            confidence = (
+                confidence_match.group(1) if confidence_match else "100"
+            )
+
+            # Simple accuracy check (for basic reporting)
+            # Note: Real evaluation would use a more sophisticated approach
+            is_correct = exact_answer.lower() == correct_answer.lower()
+            if is_correct:
+                correct_count += 1
+
+            # Format result for output
+            result = {
+                "id": example.get("id", f"q{i}"),
+                "problem": problem,
+                "correct_answer": correct_answer,
+                "response": response,
+                "extracted_answer": exact_answer,
+                "confidence": confidence,
+                "is_correct": is_correct,
+            }
+
+            # Write incrementally to output file
+            with open(output_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(result) + "\n")
+
+            results.append(result)
+
+            # Print progress
+            print(f"  Response: {exact_answer}")
+            print(f"  Correct: {is_correct}")
+            print(
+                f"  Current accuracy: {correct_count}/{i + 1} ({(correct_count / (i + 1)) * 100:.1f}%)"
+            )
+
+        except Exception as e:
+            print(f"Error processing question {i + 1}: {e!s}")
+            # In case of error, write a placeholder result
+            result = {
+                "id": example.get("id", f"q{i}"),
+                "problem": problem,
+                "correct_answer": correct_answer,
+                "response": f"Error processing this question: {str(e)[:100]}",
+                "extracted_answer": "None",
+                "confidence": "0",
+                "is_correct": False,
+            }
+            with open(output_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(result) + "\n")
+
+            results.append(result)
+
+    # Calculate overall accuracy
+    accuracy = correct_count / len(examples) if examples else 0
+
+    # Write summary report
+    report = {
+        "total_examples": len(examples),
+        "correct_count": correct_count,
+        "accuracy": accuracy,
+        "search_iterations": search_iterations,
+        "questions_per_iteration": questions_per_iteration,
+        "search_tool": search_tool,
+    }
+
+    report_path = str(Path(output_dir) / "browsecomp_summary.json")
+    with open(report_path, "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+
+    print("\nEvaluation complete.")
+    print(f"Results saved to {output_path}")
+    print(f"Summary saved to {report_path}")
+    print(f"Final accuracy: {accuracy:.4f} ({correct_count}/{len(examples)})")
+
+    return results
+
+
+# Main execution
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run BrowseComp benchmark with proper decryption"
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=10,
+        help="Number of examples to use (default: 10)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=2,
+        help="Search iterations (default: 2)",
+    )
+    parser.add_argument(
+        "--questions",
+        type=int,
+        default=9,
+        help="Questions per iteration (default: 9)",
+    )
+    parser.add_argument(
+        "--search-tool",
+        type=str,
+        default="searxng",
+        help="Search tool to use (default: searxng)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="benchmark_results/browsecomp",
+        help="Output directory (default: benchmark_results/browsecomp)",
+    )
+
+    args = parser.parse_args()
+
+    print("Starting BrowseComp benchmark with proper decryption...")
+    run_browsecomp_evaluation(
+        num_examples=args.examples,
+        search_iterations=args.iterations,
+        questions_per_iteration=args.questions,
+        search_tool=args.search_tool,
+        output_dir=args.output_dir,
+    )

examples/benchmarks/browsecomp_benchmark_readme.md

@@ -0,0 +1,84 @@
+# BrowseComp Benchmark for Local Deep Research
+
+This document explains how to run the BrowseComp benchmark with Local Deep Research.
+
+## Overview
+
+BrowseComp is a benchmark created by OpenAI to evaluate models on their ability to understand complex questions that may require browsing the web for answers. The questions in BrowseComp often involve multiple criteria that must be satisfied by a single answer.
+
+The benchmark questions are initially provided in an encrypted format, which requires decryption using a "canary" field in the dataset.
+
+## Running the Benchmark
+
+We've created a script called `browsecomp_fixed.py` in the root directory that properly decrypts the BrowseComp questions and runs them through Local Deep Research.
+
+### Basic Usage
+
+```bash
+python /path/to/browsecomp_fixed.py --examples 5 --iterations 3 --questions 3
+```
+
+### Parameters
+
+- `--examples`: Number of examples to run (default: 5)
+- `--iterations`: Number of search iterations per query (default: 1)
+- `--questions`: Questions per iteration (default: 1)
+- `--search-tool`: Search tool to use (default: "searxng")
+- `--output-dir`: Directory to save results (default: "browsecomp_results")
+
+### Performance Tips
+
+BrowseComp questions are challenging and may require more thorough search strategies:
+
+1. **Use more iterations**: Increasing the number of iterations allows the system to refine its understanding of the question and explore different aspects.
+   ```bash
+   python browsecomp_fixed.py --iterations 5
+   ```
+
+2. **Increase questions per iteration**: More questions means more angles to explore.
+   ```bash
+   python browsecomp_fixed.py --questions 5
+   ```
+
+3. **Try different search engines**: Some search engines might perform better for certain types of questions.
+   ```bash
+   python browsecomp_fixed.py --search-tool wikipedia
+   ```
+
+4. **Combine parameters for best results**:
+   ```bash
+   python browsecomp_fixed.py --examples 10 --iterations 3 --questions 3 --search-tool searxng
+   ```
+
+## Understanding the Results
+
+The script saves results in the output directory with the following files:
+
+- `browsecomp_[timestamp]_results.jsonl`: Raw results from the benchmark
+- `browsecomp_[timestamp]_evaluation.jsonl`: Evaluation of the results by a grader model
+
+The script will also print a summary with:
+- Overall accuracy
+- Number of correct answers
+- Average processing time
+
+## How It Works
+
+1. The script loads the BrowseComp dataset
+2. For each example, it decrypts the problem and correct answer using the "canary" field
+3. It runs the decrypted question through LDR's search system
+4. It extracts the answer from LDR's response
+5. A grader evaluates whether the extracted answer matches the correct answer
+
+## Improving Benchmark Performance
+
+To improve performance on the BrowseComp benchmark:
+
+1. Optimize parameters for more thorough search (more iterations and questions)
+2. Use search engines that provide more relevant results for complex queries
+3. Consider pre-processing or reformulating the questions to better match search engine capabilities
+4. Experiment with different search strategies in LDR's configuration
+
+## Technical Details
+
+The decryption process uses a simple XOR cipher with a key derived from the canary value using SHA-256. This matches the approach used in the original BrowseComp evaluation script.

examples/benchmarks/claude_grading/README.md

@@ -0,0 +1,43 @@
+# Claude API Grading Benchmark
+
+This benchmark integrates Claude 3 Sonnet for grading benchmark results with proper API access through the local database.
+
+## Features
+
+- Uses Claude 3 Sonnet for grading benchmark results
+- Accesses API keys from the local database
+- Supports SimpleQA and BrowseComp benchmarks
+- Provides composite scoring with customizable weights
+- Comprehensive metrics and accuracy reports
+
+## Usage
+
+From the project root directory:
+
+```bash
+# Run with default settings (source_based strategy, 1 iteration, 5 examples)
+./examples/benchmarks/claude_grading/run_benchmark.sh
+
+# Run with custom parameters
+./examples/benchmarks/claude_grading/run_benchmark.sh --strategy source_based --iterations 2 --examples 200
+```
+
+## How It Works
+
+The benchmark integrates with the evaluation system by patching the grading module to use the local `get_llm` function, which properly retrieves API keys from the database and configures the Claude model for grading.
+
+This approach ensures accurate grading of benchmark results and enables comparison between different strategies and configurations.
+
+## Requirements
+
+- Valid Claude API key stored in the local database
+- SearXNG search engine running locally
+- Python dependencies installed
+
+## Output
+
+Results are saved in the `benchmark_results` directory with comprehensive metrics:
+- Accuracy scores
+- Processing times
+- Grading confidence
+- Detailed evaluation reports

examples/benchmarks/claude_grading/__init__.py

@@ -0,0 +1,6 @@
+"""
+Claude API Grading Benchmark Module
+
+This module provides tools for benchmarking search strategies
+with proper grading through Claude API integration.
+"""

examples/benchmarks/claude_grading/benchmark.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+"""
+Benchmark with Claude API Grading Integration
+
+This script runs a comprehensive evaluation of search strategies with
+proper Claude API integration for grading benchmark results.
+
+Features:
+- Uses the local database for API keys
+- Configures Claude 3 Sonnet for grading
+- Supports SimpleQA and BrowseComp evaluations
+- Provides detailed metrics and accuracy reports
+"""
+
+import os
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+
+
+# Set up Python path
+src_dir = str((Path(__file__).parent / "src").resolve())
+if src_dir not in sys.path:
+    sys.path.insert(0, src_dir)
+
+# Note: Database configuration is now per-user
+# For benchmarks, API keys should be provided via environment variables
+# or configuration files rather than relying on a shared database
+
+# Logger is already imported from loguru
+
+
+def setup_grading_config():
+    """
+    Create a custom evaluation configuration that uses environment variables
+    for API keys and specifically uses Claude 3 Sonnet for grading.
+
+    Returns:
+        Dict containing the evaluation configuration
+    """
+
+    # Create config that uses Claude 3 Sonnet via Anthropic directly
+    # Only use parameters that get_llm() accepts
+    evaluation_config = {
+        "model_name": "claude-3-sonnet-20240229",  # Correct Anthropic model name
+        "provider": "anthropic",  # Use Anthropic directly
+        "temperature": 0,  # Zero temp for consistent evaluation
+    }
+
+    # Check if anthropic API key is available in environment
+    anthropic_key = os.environ.get("ANTHROPIC_API_KEY")
+    if anthropic_key:
+        print(
+            "Found Anthropic API key in environment, will use Claude 3 Sonnet for grading"
+        )
+    else:
+        print(
+            "Warning: No Anthropic API key found in ANTHROPIC_API_KEY environment variable"
+        )
+        print("Checking for alternative providers...")
+
+        # Try OpenRouter as a fallback
+        openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+        if openrouter_key:
+            print(
+                "Found OpenRouter API key, will use OpenRouter with Claude 3 Sonnet"
+            )
+            evaluation_config = {
+                "model_name": "anthropic/claude-3-sonnet-20240229",  # OpenRouter format
+                "provider": "openai_endpoint",
+                "openai_endpoint_url": "https://openrouter.ai/api/v1",
+                "temperature": 0,
+            }
+        else:
+            print("ERROR: No API keys found in environment variables")
+            print("Please set either ANTHROPIC_API_KEY or OPENROUTER_API_KEY")
+            return None
+
+    return evaluation_config
+
+
+def run_benchmark(strategy="source_based", iterations=1, examples=5):
+    """
+    Run a comprehensive benchmark evaluation of a specific strategy configuration.
+
+    Args:
+        strategy: Search strategy to evaluate (default: source_based)
+        iterations: Number of iterations for the strategy (default: 1)
+        examples: Number of examples to evaluate (default: 5)
+    """
+    # Import the benchmark components
+    try:
+        from local_deep_research.benchmarks.evaluators.browsecomp import (
+            BrowseCompEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.composite import (
+            CompositeBenchmarkEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.simpleqa import (
+            SimpleQAEvaluator,
+        )
+        from local_deep_research.config.llm_config import get_llm
+    except ImportError as e:
+        print(f"Error importing benchmark components: {e}")
+        print("Current sys.path:", sys.path)
+        return
+
+    # Set up custom grading configuration
+    evaluation_config = setup_grading_config()
+    if not evaluation_config:
+        print(
+            "Failed to setup evaluation configuration, proceeding with default config"
+        )
+
+    # Patch the graders module to use our local get_llm
+    try:
+        # This ensures we use the local get_llm function that accesses the database
+        import local_deep_research.benchmarks.graders as graders
+
+        # Store the original function for reference
+        original_get_evaluation_llm = graders.get_evaluation_llm
+
+        # Define a new function that uses our local get_llm directly
+        def custom_get_evaluation_llm(custom_config=None):
+            """
+            Override that uses the local get_llm with database access.
+            """
+            if custom_config is None:
+                custom_config = evaluation_config
+
+            print(f"Getting evaluation LLM with config: {custom_config}")
+            return get_llm(**custom_config)
+
+        # Replace the function with our custom version
+        graders.get_evaluation_llm = custom_get_evaluation_llm
+        print(
+            "Successfully patched graders.get_evaluation_llm to use local get_llm function"
+        )
+
+    except Exception as e:
+        print(f"Error patching graders module: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Create timestamp for output
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    output_dir = str(Path("benchmark_results") / f"claude_grading_{timestamp}")
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+    config = {
+        "search_strategy": strategy,
+        "iterations": iterations,
+        # Add other fixed parameters to ensure a complete run
+        "questions_per_iteration": 1,
+        "max_results": 10,
+        "search_tool": "searxng",  # Specify SearXNG search engine
+        "timeout": 10,  # Very short timeout to speed up the demo
+    }
+
+    # Run SimpleQA benchmark
+    print(
+        f"\n=== Running SimpleQA benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    simpleqa_start = time.time()
+
+    try:
+        # Create SimpleQA evaluator (without the evaluation_config parameter)
+        simpleqa = SimpleQAEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        simpleqa_results = simpleqa.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "simpleqa"),
+        )
+
+        simpleqa_duration = time.time() - simpleqa_start
+        print(
+            f"SimpleQA evaluation complete in {simpleqa_duration:.1f} seconds"
+        )
+        print(f"SimpleQA accuracy: {simpleqa_results.get('accuracy', 0):.4f}")
+        print(f"SimpleQA metrics: {simpleqa_results.get('metrics', {})}")
+
+        # Save results
+        import json
+
+        with open(
+            Path(output_dir) / "simpleqa_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(simpleqa_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during SimpleQA evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run BrowseComp benchmark
+    print(
+        f"\n=== Running BrowseComp benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    browsecomp_start = time.time()
+
+    try:
+        # Create BrowseComp evaluator (without the evaluation_config parameter)
+        browsecomp = BrowseCompEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        browsecomp_results = browsecomp.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "browsecomp"),
+        )
+
+        browsecomp_duration = time.time() - browsecomp_start
+        print(
+            f"BrowseComp evaluation complete in {browsecomp_duration:.1f} seconds"
+        )
+        print(f"BrowseComp score: {browsecomp_results.get('score', 0):.4f}")
+        print(f"BrowseComp metrics: {browsecomp_results.get('metrics', {})}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "browsecomp_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(browsecomp_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during BrowseComp evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run composite benchmark
+    print(
+        f"\n=== Running Composite benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    composite_start = time.time()
+
+    try:
+        # Create composite evaluator with benchmark weights (without evaluation_config parameter)
+        benchmark_weights = {"simpleqa": 0.5, "browsecomp": 0.5}
+        composite = CompositeBenchmarkEvaluator(
+            benchmark_weights=benchmark_weights
+        )
+        composite_results = composite.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "composite"),
+        )
+
+        composite_duration = time.time() - composite_start
+        print(
+            f"Composite evaluation complete in {composite_duration:.1f} seconds"
+        )
+        print(f"Composite score: {composite_results.get('score', 0):.4f}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "composite_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(composite_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during composite evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Generate summary
+    print("\n=== Evaluation Summary ===")
+    print(f"Strategy: {strategy}")
+    print(f"Iterations: {iterations}")
+    print(f"Examples: {examples}")
+    print(f"Results saved to: {output_dir}")
+
+    # If we patched the graders module, restore the original function
+    if "original_get_evaluation_llm" in locals():
+        graders.get_evaluation_llm = original_get_evaluation_llm
+        print("Restored original graders.get_evaluation_llm function")
+
+    return {
+        "simpleqa": simpleqa_results
+        if "simpleqa_results" in locals()
+        else None,
+        "browsecomp": browsecomp_results
+        if "browsecomp_results" in locals()
+        else None,
+        "composite": composite_results
+        if "composite_results" in locals()
+        else None,
+    }
+
+
+def main():
+    # Parse command line arguments
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run benchmark with Claude API grading"
+    )
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="source_based",
+        help="Strategy to evaluate (default: source_based)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=1,
+        help="Number of iterations (default: 1)",
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=5,
+        help="Number of examples to evaluate (default: 5)",
+    )
+
+    args = parser.parse_args()
+
+    print(
+        "Starting benchmark of {} strategy with {} iterations".format(
+            args.strategy, args.iterations
+        )
+    )
+    print(f"Evaluating with {args.examples} examples")
+
+    # Run the evaluation
+    results = run_benchmark(
+        strategy=args.strategy,
+        iterations=args.iterations,
+        examples=args.examples,
+    )
+
+    # Return success if at least one benchmark completed
+    return 0 if any(results.values()) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/claude_grading/run_benchmark.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Run a benchmark with Claude API grading integration
+
+# Navigate to project root which is needed for proper imports
+cd "$(dirname "$0")/../../.." || exit 1
+echo "Changed to project root: $(pwd)"
+
+# Activate virtual environment if it exists
+VENV_PATH=".venv/bin/activate"
+if [ -f "$VENV_PATH" ]; then
+    echo "Activating virtual environment..."
+    # shellcheck source=/dev/null
+    source "$VENV_PATH"
+else
+    echo "Warning: Virtual environment not found at $VENV_PATH"
+fi
+
+# Create a timestamp for output directory
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+OUTPUT_DIR="benchmark_results/claude_benchmark_${TIMESTAMP}"
+mkdir -p "$OUTPUT_DIR"
+
+echo "Running benchmark with Claude API grading..."
+echo "Results will be saved to: $OUTPUT_DIR"
+
+# Use a long timeout for comprehensive benchmarks
+pdm run timeout 86400 python -m examples.benchmarks.claude_grading.benchmark "$@"
+
+echo "Benchmark complete or timed out."
+echo "Check $OUTPUT_DIR for results."

examples/benchmarks/gemini/README.md

@@ -0,0 +1,32 @@
+# Gemini Benchmark Examples
+
+This directory contains example scripts for running benchmarks with Gemini models via OpenRouter.
+
+## Scripts Included
+
+### run_gemini_benchmark_fixed.py
+A comprehensive benchmark script that runs both SimpleQA and BrowseComp evaluations
+using Google's Gemini 2.0 Flash model via the OpenRouter API.
+
+Key features:
+- Patches the LLM configuration to use Gemini for all evaluations
+- Supports both SimpleQA and BrowseComp benchmarks
+- Properly handles result collection and reporting
+
+## Usage
+
+To run the benchmark with Gemini:
+
+```bash
+# Run with default settings (1 example)
+python run_gemini_benchmark_fixed.py
+
+# Run with custom number of examples
+python run_gemini_benchmark_fixed.py --examples 5
+```
+
+## Notes
+
+These scripts assume you have:
+1. An OpenRouter API key configured in your LDR database
+2. The correct access permissions for the Gemini model

examples/benchmarks/gemini/run_gemini_benchmark_fixed.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+"""
+Fixed benchmark with Gemini 2.0 Flash via OpenRouter
+"""
+
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+
+# Import the benchmark functions
+from local_deep_research.benchmarks.benchmark_functions import (
+    evaluate_browsecomp,
+    evaluate_simpleqa,
+)
+
+# Monkey patch the get_llm function to use Gemini
+from local_deep_research.config import llm_config
+
+# Save original function
+original_get_llm = llm_config.get_llm
+
+
+def setup_gemini_config():
+    """
+    Create a custom evaluation configuration using Gemini 2.0 Flash via OpenRouter
+    """
+    # Configure to use Gemini 2.0 Flash via OpenRouter
+    evaluation_config = {
+        "model_name": "google/gemini-2.0-flash-001",  # OpenRouter format for Gemini
+        "provider": "openai_endpoint",  # Use OpenRouter as endpoint
+        "openai_endpoint_url": "https://openrouter.ai/api/v1",
+        "temperature": 0,  # Zero temp for consistent evaluation
+    }
+
+    print(f"Using Gemini 2.0 Flash for evaluation: {evaluation_config}")
+    return evaluation_config
+
+
+# Override get_llm to always use Gemini
+def patched_get_llm(
+    model_name=None, temperature=None, provider=None, openai_endpoint_url=None
+):
+    """Patched version that always uses Gemini via OpenRouter"""
+    if (
+        model_name == "gemma3:12b"
+    ):  # This is the default model that causes the error
+        print("Overriding local model with Gemini 2.0 Flash")
+        model_name = "google/gemini-2.0-flash-001"
+        provider = "openai_endpoint"
+        openai_endpoint_url = "https://openrouter.ai/api/v1"
+    return original_get_llm(
+        model_name, temperature, provider, openai_endpoint_url
+    )
+
+
+# Apply the patch
+llm_config.get_llm = patched_get_llm
+
+
+def run_benchmark(examples=1):
+    """Run benchmarks with Gemini 2.0 Flash"""
+    try:
+        # Create timestamp for output
+        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+        output_dir = str(
+            Path(__file__).parent.parent.parent
+            / "benchmark_results"
+            / f"gemini_eval_{timestamp}"
+        )
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+        # Setup the Gemini configuration
+        gemini_config = setup_gemini_config()
+
+        # Run SimpleQA benchmark
+        print(f"\n=== Running SimpleQA benchmark with {examples} examples ===")
+        simpleqa_start = time.time()
+
+        simpleqa_results = evaluate_simpleqa(
+            num_examples=examples,
+            search_iterations=2,
+            questions_per_iteration=3,
+            search_tool="searxng",
+            evaluation_model=gemini_config["model_name"],
+            evaluation_provider=gemini_config["provider"],
+            output_dir=str(Path(output_dir) / "simpleqa"),
+        )
+
+        simpleqa_duration = time.time() - simpleqa_start
+        print(
+            f"SimpleQA evaluation complete in {simpleqa_duration:.1f} seconds"
+        )
+        if (
+            isinstance(simpleqa_results, dict)
+            and "accuracy" in simpleqa_results
+        ):
+            print(f"SimpleQA accuracy: {simpleqa_results['accuracy']:.4f}")
+        else:
+            print("SimpleQA accuracy: N/A")
+
+        # Run BrowseComp benchmark
+        print(
+            f"\n=== Running BrowseComp benchmark with {examples} examples ==="
+        )
+        browsecomp_start = time.time()
+
+        browsecomp_results = evaluate_browsecomp(
+            num_examples=examples,
+            search_iterations=3,
+            questions_per_iteration=3,
+            search_tool="searxng",
+            evaluation_model=gemini_config["model_name"],
+            evaluation_provider=gemini_config["provider"],
+            output_dir=str(Path(output_dir) / "browsecomp"),
+        )
+
+        browsecomp_duration = time.time() - browsecomp_start
+        print(
+            f"BrowseComp evaluation complete in {browsecomp_duration:.1f} seconds"
+        )
+        if (
+            isinstance(browsecomp_results, dict)
+            and "accuracy" in browsecomp_results
+        ):
+            print(f"BrowseComp accuracy: {browsecomp_results['accuracy']:.4f}")
+        else:
+            print("BrowseComp accuracy: N/A")
+
+        # Generate summary
+        print("\n=== Evaluation Summary ===")
+        print(f"Examples: {examples}")
+        print(f"Model: {gemini_config.get('model_name', 'unknown')}")
+        print(f"Provider: {gemini_config.get('provider', 'unknown')}")
+        print(f"Results saved to: {output_dir}")
+
+        return {
+            "simpleqa": simpleqa_results,
+            "browsecomp": browsecomp_results,
+        }
+
+    except Exception as e:
+        print(f"Error running benchmark: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return None
+
+
+def main():
+    # Parse command line arguments
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run benchmark with Gemini 2.0 Flash"
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=1,
+        help="Number of examples to evaluate (default: 1)",
+    )
+
+    args = parser.parse_args()
+
+    print(
+        f"Starting benchmark with Gemini 2.0 Flash on {args.examples} examples"
+    )
+
+    # Run the evaluation
+    results = run_benchmark(examples=args.examples)
+
+    # Return success if benchmark completed
+    return 0 if results else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/run_browsecomp.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+"""
+BrowseComp benchmark with proper decryption.
+
+This script runs the BrowseComp benchmark with proper decryption using the canary field.
+
+Usage:
+    # Install dependencies with PDM
+    cd /path/to/local-deep-research
+    pdm install
+
+    # Run the script with PDM
+    pdm run python examples/benchmarks/run_browsecomp.py --help
+"""
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict
+
+from loguru import logger
+
+from local_deep_research.api import quick_summary
+from local_deep_research.benchmarks.datasets import load_dataset
+from local_deep_research.benchmarks.graders import grade_results
+from local_deep_research.benchmarks.templates import BROWSECOMP_QUERY_TEMPLATE
+
+
+def derive_key(password: str, length: int) -> bytes:
+    """Derive a fixed-length key from the password using SHA256."""
+    hasher = hashlib.sha256()
+    hasher.update(password.encode())
+    key = hasher.digest()
+    return key * (length // len(key)) + key[: length % len(key)]
+
+
+def decrypt(ciphertext_b64: str, password: str) -> str:
+    """Decrypt base64-encoded ciphertext with XOR."""
+    try:
+        encrypted = base64.b64decode(ciphertext_b64)
+        key = derive_key(password, len(encrypted))
+        decrypted = bytes(a ^ b for a, b in zip(encrypted, key, strict=False))
+        return decrypted.decode()
+    except Exception as e:
+        logger.exception("Error decrypting data")
+        return f"Error: Could not decrypt data - {e!s}"
+
+
+def run_browsecomp_with_canary(
+    num_examples: int = 5,
+    search_iterations: int = 1,
+    questions_per_iteration: int = 1,
+    search_tool: str = "searxng",
+    output_dir: str = "browsecomp_results",
+) -> Dict[str, Any]:
+    """
+    Run BrowseComp benchmark with proper decryption using canary field.
+
+    Args:
+        num_examples: Number of examples to evaluate
+        search_iterations: Number of search iterations per query
+        questions_per_iteration: Number of questions per iteration
+        search_tool: Search engine to use
+        output_dir: Directory to save results
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    # Ensure output directory exists
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Load BrowseComp dataset
+    dataset = load_dataset(
+        dataset_type="browsecomp",
+        num_examples=num_examples,
+        seed=42,
+    )
+
+    # Set up output files
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    results_file = str(
+        Path(output_dir) / f"browsecomp_{timestamp}_results.jsonl"
+    )
+    evaluation_file = str(
+        Path(output_dir) / f"browsecomp_{timestamp}_evaluation.jsonl"
+    )
+
+    # Make sure output files don't exist
+    for file in [results_file, evaluation_file]:
+        if Path(file).exists():
+            os.remove(file)
+
+    # Process each example
+    results = []
+    total_examples = len(dataset)
+
+    for i, example in enumerate(dataset):
+        # Decrypt the problem and answer using the canary
+        try:
+            encrypted_question = example.get("problem", "")
+            encrypted_answer = example.get("answer", "")
+            canary = example.get("canary", "")
+
+            # Decrypt question and answer
+            decrypted_question = decrypt(encrypted_question, canary)
+            decrypted_answer = decrypt(encrypted_answer, canary)
+
+            logger.info(
+                f"Processing {i + 1}/{total_examples}: {decrypted_question[:50]}..."
+            )
+
+            # Format query for BrowseComp
+            formatted_query = BROWSECOMP_QUERY_TEMPLATE.format(
+                question=decrypted_question
+            )
+
+            # Time the search
+            start_time = time.time()
+
+            # Get response from LDR
+            search_result = quick_summary(
+                query=formatted_query,
+                iterations=search_iterations,
+                questions_per_iteration=questions_per_iteration,
+                search_tool=search_tool,
+            )
+
+            end_time = time.time()
+            processing_time = end_time - start_time
+
+            # Extract response
+            response = search_result.get("summary", "")
+
+            # Extract exact answer from the response
+            answer_match = re.search(r"Exact Answer:\s*(.*?)(?:\n|$)", response)
+            exact_answer = (
+                answer_match.group(1).strip() if answer_match else "None"
+            )
+
+            # Extract confidence from the response
+            confidence_match = re.search(r"Confidence:\s*(\d+)%", response)
+            confidence = (
+                confidence_match.group(1) if confidence_match else "100"
+            )
+
+            # Format result
+            result = {
+                "id": example.get("id", f"example_{i}"),
+                "problem": decrypted_question,  # Store decrypted question
+                "correct_answer": decrypted_answer,  # Store decrypted answer
+                "response": response,
+                "extracted_answer": exact_answer,
+                "confidence": confidence,
+                "processing_time": processing_time,
+                "sources": search_result.get("sources", []),
+                "search_config": {
+                    "iterations": search_iterations,
+                    "questions_per_iteration": questions_per_iteration,
+                    "search_tool": search_tool,
+                },
+            }
+
+            # Add to results list
+            results.append(result)
+
+            # Write result to file
+            with open(results_file, "a", encoding="utf-8") as f:
+                f.write(json.dumps(result) + "\n")
+
+        except Exception as e:
+            logger.exception(f"Error processing example {i + 1}")
+
+            # Create error result
+            error_result = {
+                "id": example.get("id", f"example_{i}"),
+                "problem": (
+                    decrypted_question
+                    if "decrypted_question" in locals()
+                    else "Error: Could not decrypt problem"
+                ),
+                "correct_answer": (
+                    decrypted_answer
+                    if "decrypted_answer" in locals()
+                    else "Error: Could not decrypt answer"
+                ),
+                "error": str(e),
+                "processing_time": (
+                    time.time() - start_time if "start_time" in locals() else 0
+                ),
+            }
+
+            # Add to results list
+            results.append(error_result)
+
+            # Write error result to file
+            with open(results_file, "a", encoding="utf-8") as f:
+                f.write(json.dumps(error_result) + "\n")
+
+    logger.info(f"Completed processing {total_examples} examples")
+
+    # Run evaluation
+    logger.info("Running automated evaluation...")
+    try:
+        evaluation_results = grade_results(
+            results_file=results_file,
+            output_file=evaluation_file,
+            dataset_type="browsecomp",
+        )
+    except Exception:
+        logger.exception("Evaluation failed")
+        evaluation_results = []
+
+    # Calculate basic metrics
+    correct_count = sum(
+        1 for result in evaluation_results if result.get("is_correct", False)
+    )
+    accuracy = correct_count / len(results) if results else 0
+    avg_time = (
+        sum(result.get("processing_time", 0) for result in results)
+        / len(results)
+        if results
+        else 0
+    )
+
+    print("\nBrowseComp Benchmark Results:")
+    print(f"  Accuracy: {accuracy:.3f}")
+    print(f"  Total examples: {total_examples}")
+    print(f"  Correct answers: {correct_count}")
+    print(f"  Average time: {avg_time:.2f}s")
+    print()
+    print(f"Report saved to: {evaluation_file}")
+
+    return {
+        "status": "complete",
+        "dataset_type": "browsecomp",
+        "results_path": results_file,
+        "evaluation_path": evaluation_file,
+        "metrics": {"accuracy": accuracy, "average_processing_time": avg_time},
+        "total_examples": total_examples,
+        "accuracy": accuracy,
+    }
+
+
+def main():
+    """Run the BrowseComp benchmark with command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Run BrowseComp benchmark with proper decryption"
+    )
+    parser.add_argument(
+        "--examples", type=int, default=2, help="Number of examples to run"
+    )
+    parser.add_argument(
+        "--iterations", type=int, default=1, help="Number of search iterations"
+    )
+    parser.add_argument(
+        "--questions", type=int, default=1, help="Questions per iteration"
+    )
+    parser.add_argument(
+        "--search-tool", type=str, default="searxng", help="Search tool to use"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path("examples") / "benchmarks" / "results" / "browsecomp"),
+        help="Output directory",
+    )
+
+    args = parser.parse_args()
+
+    run_browsecomp_with_canary(
+        num_examples=args.examples,
+        search_iterations=args.iterations,
+        questions_per_iteration=args.questions,
+        search_tool=args.search_tool,
+        output_dir=args.output_dir,
+    )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/run_gemini_benchmark.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+"""
+Gemini Benchmark Runner for Local Deep Research.
+
+This script provides a convenient way to run benchmarks with Gemini via OpenRouter.
+
+Usage:
+    # Install dependencies with PDM
+    cd /path/to/local-deep-research
+    pdm install
+
+    # Run the script with PDM and your OpenRouter API key
+    pdm run python examples/benchmarks/run_gemini_benchmark.py --api-key YOUR_API_KEY
+"""
+
+import argparse
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+
+# Import the benchmark functionality
+from local_deep_research.benchmarks.benchmark_functions import (
+    evaluate_browsecomp,
+    evaluate_simpleqa,
+)
+
+
+def setup_gemini_config(api_key):
+    """
+    Create a configuration for using Gemini via OpenRouter.
+
+    Args:
+        api_key: OpenRouter API key
+
+    Returns:
+        Dictionary with configuration settings
+    """
+    return {
+        "model_name": "google/gemini-2.0-flash-001",
+        "provider": "openai_endpoint",
+        "openai_endpoint_url": "https://openrouter.ai/api/v1",
+        "api_key": api_key,
+    }
+
+
+def run_benchmark(args):
+    """
+    Run benchmarks with Gemini via OpenRouter.
+
+    Args:
+        args: Command line arguments
+    """
+    # Set up configuration
+    config = setup_gemini_config(args.api_key)
+
+    # Set environment variables
+    if args.api_key:
+        os.environ["OPENAI_ENDPOINT_API_KEY"] = args.api_key
+        os.environ["LDR_LLM__OPENAI_ENDPOINT_API_KEY"] = args.api_key
+    os.environ["OPENAI_ENDPOINT_URL"] = config["openai_endpoint_url"]
+    os.environ["LDR_LLM__OPENAI_ENDPOINT_URL"] = config["openai_endpoint_url"]
+
+    # Create timestamp for output directory
+    from datetime import timezone
+
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    base_output_dir = str(
+        Path("examples") / "benchmarks" / "results" / f"gemini_{timestamp}"
+    )
+    Path(base_output_dir).mkdir(parents=True, exist_ok=True)
+
+    # Configure benchmark settings
+    results = {}
+    benchmarks = []
+
+    if args.simpleqa:
+        benchmarks.append(
+            {
+                "name": "SimpleQA",
+                "function": evaluate_simpleqa,
+                "output_dir": str(Path(base_output_dir) / "simpleqa"),
+            }
+        )
+
+    if args.browsecomp:
+        benchmarks.append(
+            {
+                "name": "BrowseComp",
+                "function": evaluate_browsecomp,
+                "output_dir": str(Path(base_output_dir) / "browsecomp"),
+            }
+        )
+
+    # Run selected benchmarks
+    for benchmark in benchmarks:
+        print(
+            f"\n=== Running {benchmark['name']} benchmark with {args.examples} examples ==="
+        )
+        start_time = time.time()
+
+        benchmark_result = benchmark["function"](
+            num_examples=args.examples,
+            search_iterations=args.iterations,
+            questions_per_iteration=args.questions,
+            search_tool=args.search_tool,
+            search_model=config["model_name"],
+            search_provider=config["provider"],
+            endpoint_url=config["openai_endpoint_url"],
+            search_strategy=args.search_strategy,
+            evaluation_model=config["model_name"],
+            evaluation_provider=config["provider"],
+            output_dir=benchmark["output_dir"],
+        )
+
+        duration = time.time() - start_time
+        print(
+            f"{benchmark['name']} evaluation complete in {duration:.1f} seconds"
+        )
+
+        if (
+            isinstance(benchmark_result, dict)
+            and "accuracy" in benchmark_result
+        ):
+            print(
+                f"{benchmark['name']} accuracy: {benchmark_result['accuracy']:.4f}"
+            )
+        else:
+            print(f"{benchmark['name']} accuracy: N/A")
+
+        results[benchmark["name"].lower()] = benchmark_result
+
+    # Print summary
+    print("\n=== Benchmark Summary ===")
+    print(f"Model: {config['model_name']}")
+    print(f"Provider: {config['provider']}")
+    print(f"Examples: {args.examples}")
+    print(f"Results saved to: {base_output_dir}")
+
+    return results
+
+
+def main():
+    """Parse arguments and run the benchmark."""
+    parser = argparse.ArgumentParser(
+        description="Run benchmarks with Gemini via OpenRouter"
+    )
+
+    # API key is required
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        required=True,
+        help="OpenRouter API key (required)",
+    )
+
+    # Benchmark selection (at least one required)
+    benchmark_group = parser.add_argument_group("benchmark selection")
+    benchmark_group.add_argument(
+        "--simpleqa", action="store_true", help="Run SimpleQA benchmark"
+    )
+    benchmark_group.add_argument(
+        "--browsecomp", action="store_true", help="Run BrowseComp benchmark"
+    )
+
+    # Benchmark parameters
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=3,
+        help="Number of examples to run (default: 3)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=2,
+        help="Number of search iterations (default: 2)",
+    )
+    parser.add_argument(
+        "--questions",
+        type=int,
+        default=3,
+        help="Questions per iteration (default: 3)",
+    )
+    parser.add_argument(
+        "--search-tool",
+        type=str,
+        default="searxng",
+        help="Search tool to use (default: searxng)",
+    )
+    parser.add_argument(
+        "--search-strategy",
+        type=str,
+        default="source_based",
+        choices=["source_based", "standard", "rapid", "parallel", "iterdrag"],
+        help="Search strategy to use (default: source_based)",
+    )
+
+    args = parser.parse_args()
+
+    # Ensure at least one benchmark is selected
+    if not (args.simpleqa or args.browsecomp):
+        parser.error(
+            "At least one benchmark must be selected (--simpleqa or --browsecomp)"
+        )
+
+    print(
+        f"Starting benchmarks with Gemini 2.0 Flash on {args.examples} examples"
+    )
+    run_benchmark(args)
+
+
+if __name__ == "__main__":
+    main()

examples/benchmarks/run_resumable_parallel_benchmark.py

@@ -0,0 +1,583 @@
+#!/usr/bin/env python
+"""
+Run SimpleQA and BrowseComp benchmarks in parallel with resume capability.
+
+This script can resume interrupted benchmarks by reading existing results
+and continuing from where it left off.
+
+Usage:
+    # Start new benchmark
+    pdm run python examples/benchmarks/run_resumable_parallel_benchmark.py
+
+    # Resume interrupted benchmark
+    pdm run python examples/benchmarks/run_resumable_parallel_benchmark.py \
+        --resume-from benchmark_results/parallel_benchmark_20250513_235221
+"""
+
+import argparse
+import concurrent.futures
+import json
+import os
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+from loguru import logger
+
+from local_deep_research.api import quick_summary
+from local_deep_research.benchmarks.datasets import load_dataset
+from local_deep_research.benchmarks.graders import (
+    extract_answer_from_response,
+    grade_results,
+)
+from local_deep_research.benchmarks.metrics import (
+    calculate_metrics,
+    generate_report,
+)
+from local_deep_research.benchmarks.runners import format_query
+
+# Add the src directory to the Python path
+project_root = str(Path(__file__).parent.parent.parent.resolve())
+
+logger.enable("local_deep_research")
+
+
+def load_existing_results(results_file: str) -> Dict[str, Dict]:
+    """Load existing results from JSONL file."""
+    results = {}
+    if Path(results_file).exists():
+        logger.info(f"Loading existing results from: {results_file}")
+        with open(results_file, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        result = json.loads(line)
+                        # Use ID field as key
+                        result_id = result.get("id", "")
+                        if result_id:
+                            results[result_id] = result
+                    except json.JSONDecodeError:
+                        logger.warning(
+                            f"Skipping invalid JSON line: {line[:50]}..."
+                        )
+        logger.info(f"Loaded {len(results)} existing results")
+    return results
+
+
+def find_latest_results_file(
+    output_dir: str, dataset_type: str
+) -> Optional[str]:
+    """Find the most recent results file for a dataset."""
+    # First try dataset subdirectory
+    dataset_dir = str(Path(output_dir) / dataset_type)
+    if Path(dataset_dir).exists():
+        pattern = f"{dataset_type}_*_results.jsonl"
+        files = list(Path(dataset_dir).glob(pattern))
+        if files:
+            # Sort by filename (includes timestamp) and return the latest
+            return str(sorted(files)[-1])
+
+    # Then try root directory
+    pattern = f"{dataset_type}_*_results.jsonl"
+    files = list(Path(output_dir).glob(pattern))
+    if files:
+        return str(sorted(files)[-1])
+
+    return None
+
+
+def run_resumable_benchmark(
+    dataset_type: str,
+    num_examples: int,
+    output_dir: str,
+    search_config: Dict[str, Any],
+    evaluation_config: Optional[Dict[str, Any]] = None,
+    resume_from: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Run a benchmark with resume capability."""
+
+    # Create output directory if needed
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Load dataset
+    dataset = load_dataset(
+        dataset_type=dataset_type,
+        num_examples=num_examples,
+        seed=None,  # Random seed for truly random sampling
+    )
+
+    # Determine output files
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    results_file = str(
+        Path(output_dir) / f"{dataset_type}_{timestamp}_results.jsonl"
+    )
+    evaluation_file = str(
+        Path(output_dir) / f"{dataset_type}_{timestamp}_evaluation.jsonl"
+    )
+    report_file = str(
+        Path(output_dir) / f"{dataset_type}_{timestamp}_report.md"
+    )
+
+    # Load existing results if resuming
+    existing_results = {}
+    if resume_from:
+        existing_results_file = find_latest_results_file(
+            resume_from, dataset_type
+        )
+        if existing_results_file:
+            existing_results = load_existing_results(existing_results_file)
+            logger.info(
+                f"Found {len(existing_results)} existing results for {dataset_type}"
+            )
+
+    # Process examples
+    all_results = []
+    new_results_count = 0
+    reused_results_count = 0
+    error_count = 0
+
+    for i, example in enumerate(dataset):
+        # Extract ID and question
+        example_id = example.get("id", f"example_{i}")
+
+        # Extract question and answer based on dataset type
+        if dataset_type.lower() == "simpleqa":
+            question = example.get("problem", "")
+            correct_answer = example.get("answer", "")
+        else:  # browsecomp
+            question = example.get("problem", "")
+            correct_answer = example.get("correct_answer", "") or example.get(
+                "answer", ""
+            )
+
+        # Check if we have existing result
+        existing_result = existing_results.get(example_id)
+
+        if existing_result and existing_result.get("response"):
+            # Reuse existing result
+            logger.info(
+                f"Reusing existing result for example {i + 1}/{len(dataset)}: {example_id}"
+            )
+            all_results.append(existing_result)
+            reused_results_count += 1
+
+            # Write to new results file
+            with open(results_file, "a", encoding="utf-8") as f:
+                f.write(json.dumps(existing_result) + "\n")
+        else:
+            # Process new example
+            logger.info(
+                f"Processing new example {i + 1}/{len(dataset)}: {question[:50]}..."
+            )
+
+            try:
+                # Format query
+                formatted_query = format_query(question, dataset_type)
+
+                # Time the search
+                start_time = time.time()
+
+                # Get response from LDR
+                search_result = quick_summary(
+                    query=formatted_query,
+                    iterations=search_config.get("iterations", 3),
+                    questions_per_iteration=search_config.get(
+                        "questions_per_iteration", 3
+                    ),
+                    search_tool=search_config.get("search_tool", "searxng"),
+                    search_strategy=search_config.get(
+                        "search_strategy", "source_based"
+                    ),
+                )
+
+                processing_time = time.time() - start_time
+
+                # Extract response
+                response = search_result.get("summary", "")
+                extracted = extract_answer_from_response(response, dataset_type)
+
+                # Create result
+                result = {
+                    "id": example_id,
+                    "problem": question,
+                    "correct_answer": correct_answer,
+                    "response": response,
+                    "extracted_answer": extracted["extracted_answer"],
+                    "confidence": extracted["confidence"],
+                    "processing_time": processing_time,
+                    "sources": search_result.get("sources", []),
+                    "search_config": search_config,
+                }
+
+                all_results.append(result)
+                new_results_count += 1
+
+                # Write to file immediately
+                with open(results_file, "a", encoding="utf-8") as f:
+                    f.write(json.dumps(result) + "\n")
+
+            except Exception as e:
+                logger.exception("Error processing example")
+                error_count += 1
+
+                # Create error result
+                error_result = {
+                    "id": example_id,
+                    "problem": question,
+                    "correct_answer": correct_answer,
+                    "error": str(e),
+                    "processing_time": 0,
+                }
+
+                all_results.append(error_result)
+                new_results_count += 1
+
+                # Write error result
+                with open(results_file, "a", encoding="utf-8") as f:
+                    f.write(json.dumps(error_result) + "\n")
+
+    logger.info(
+        f"Completed {dataset_type}: {new_results_count} new, {reused_results_count} reused, {error_count} errors"
+    )
+
+    # Run evaluation on all results
+    logger.info(f"Running evaluation for {dataset_type}")
+    try:
+        evaluation_results = grade_results(
+            results_file=results_file,
+            output_file=evaluation_file,
+            dataset_type=dataset_type,
+            evaluation_config=evaluation_config,
+        )
+        logger.info(
+            f"Evaluation results for {dataset_type}: {evaluation_results}"
+        )
+
+        # Calculate metrics
+        metrics = calculate_metrics(evaluation_file)
+        logger.info(f"Metrics for {dataset_type}: {metrics}")
+
+        # Generate report
+        generate_report(metrics, evaluation_file, report_file, dataset_type)
+
+        return {
+            "accuracy": metrics.get("accuracy", 0),
+            "metrics": metrics,
+            "new_results": new_results_count,
+            "reused_results": reused_results_count,
+            "total_results": len(all_results),
+            "errors": error_count,
+        }
+    except Exception as e:
+        logger.exception("Error during evaluation")
+        return {
+            "accuracy": 0,
+            "metrics": {},
+            "new_results": new_results_count,
+            "reused_results": reused_results_count,
+            "total_results": len(all_results),
+            "errors": error_count,
+            "evaluation_error": str(e),
+        }
+
+
+def run_simpleqa_benchmark_wrapper(args: Tuple) -> Dict[str, Any]:
+    """Wrapper for running SimpleQA benchmark in parallel."""
+    num_examples, output_dir, resume_from, search_config, evaluation_config = (
+        args
+    )
+
+    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
+    start_time = time.time()
+
+    results = run_resumable_benchmark(
+        dataset_type="simpleqa",
+        num_examples=num_examples,
+        output_dir=str(Path(output_dir) / "simpleqa"),
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        resume_from=resume_from,
+    )
+
+    duration = time.time() - start_time
+    logger.info(f"SimpleQA benchmark completed in {duration:.1f} seconds")
+
+    return results
+
+
+def run_browsecomp_benchmark_wrapper(args: Tuple) -> Dict[str, Any]:
+    """Wrapper for running BrowseComp benchmark in parallel."""
+    num_examples, output_dir, resume_from, search_config, evaluation_config = (
+        args
+    )
+
+    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
+    start_time = time.time()
+
+    # BrowseComp needs more iterations
+    browsecomp_config = {**search_config, "iterations": 3}
+
+    results = run_resumable_benchmark(
+        dataset_type="browsecomp",
+        num_examples=num_examples,
+        output_dir=str(Path(output_dir) / "browsecomp"),
+        search_config=browsecomp_config,
+        evaluation_config=evaluation_config,
+        resume_from=resume_from,
+    )
+
+    duration = time.time() - start_time
+    logger.info(f"BrowseComp benchmark completed in {duration:.1f} seconds")
+
+    return results
+
+
+def setup_llm_environment(
+    model=None, provider=None, endpoint_url=None, api_key=None
+):
+    """Set up environment variables for LLM configuration."""
+    if model:
+        os.environ["LDR_LLM_MODEL"] = model
+        logger.info(f"Using LLM model: {model}")
+
+    if provider:
+        os.environ["LDR_LLM_PROVIDER"] = provider
+        logger.info(f"Using LLM provider: {provider}")
+
+    if endpoint_url:
+        os.environ["OPENAI_ENDPOINT_URL"] = endpoint_url
+        os.environ["LDR_LLM_OPENAI_ENDPOINT_URL"] = endpoint_url
+        os.environ["LDR_LLM_OLLAMA_URL"] = endpoint_url
+        logger.info(f"Using endpoint URL: {endpoint_url}")
+
+    if api_key:
+        # Set the appropriate environment variable based on provider
+        if provider == "openai":
+            os.environ["OPENAI_API_KEY"] = api_key
+            os.environ["LDR_LLM_OPENAI_API_KEY"] = api_key
+        elif provider == "openai_endpoint":
+            os.environ["OPENAI_ENDPOINT_API_KEY"] = api_key
+            os.environ["LDR_LLM_OPENAI_ENDPOINT_API_KEY"] = api_key
+        elif provider == "anthropic":
+            os.environ["ANTHROPIC_API_KEY"] = api_key
+            os.environ["LDR_LLM_ANTHROPIC_API_KEY"] = api_key
+
+        logger.info("API key configured")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Run SimpleQA and BrowseComp benchmarks in parallel with resume capability"
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=20,
+        help="Number of examples for each benchmark (default: 20)",
+    )
+    parser.add_argument(
+        "--resume-from",
+        help="Path to previous benchmark results directory to resume from",
+    )
+
+    # LLM configuration options
+    parser.add_argument(
+        "--model",
+        help="Model name for the LLM (e.g., 'google/gemini-2.0-flash-001')",
+    )
+    parser.add_argument(
+        "--provider",
+        help="Provider for the LLM (e.g., 'anthropic', 'openai', 'openai_endpoint')",
+    )
+    parser.add_argument(
+        "--endpoint-url",
+        help="Custom endpoint URL (e.g., 'https://openrouter.ai/api/v1')",
+    )
+    parser.add_argument("--api-key", help="API key for the LLM provider")
+    parser.add_argument(
+        "--datasets",
+        choices=["simpleqa", "browsecomp", "both"],
+        default="both",
+        help="Which datasets to run (default: both)",
+    )
+
+    args = parser.parse_args()
+
+    # Determine output directory
+    if args.resume_from:
+        # Create new directory but link to old results
+        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+        output_dir = str(
+            Path(project_root)
+            / "benchmark_results"
+            / f"resumed_benchmark_{timestamp}"
+        )
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        logger.info(
+            f"Resuming from {args.resume_from}, new results in {output_dir}"
+        )
+    else:
+        # Create new timestamp directory
+        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+        output_dir = str(
+            Path(project_root)
+            / "benchmark_results"
+            / f"parallel_benchmark_{timestamp}"
+        )
+        os.makedirs(output_dir, exist_ok=True)
+        logger.info(f"Starting new benchmark in: {output_dir}")
+
+    # Display start information
+    print(f"Starting parallel benchmarks with {args.examples} examples each")
+    print(f"Results will be saved to: {output_dir}")
+    if args.resume_from:
+        print(f"Resuming from previous run: {args.resume_from}")
+
+    # Set up LLM environment if specified
+    setup_llm_environment(
+        model=args.model,
+        provider=args.provider,
+        endpoint_url=args.endpoint_url,
+        api_key=args.api_key,
+    )
+
+    # Set up configurations
+    search_config = {
+        "iterations": 8,  # Same as original 96% benchmark
+        "questions_per_iteration": 5,  # Same as original 96% benchmark
+        "search_tool": "searxng",
+        "search_strategy": "focused_iteration",  # Same as original 96% benchmark
+        # performance
+    }
+
+    # Add model configurations if provided
+    if args.model:
+        search_config["model_name"] = args.model
+    if args.provider:
+        search_config["provider"] = args.provider
+    if args.endpoint_url:
+        search_config["openai_endpoint_url"] = args.endpoint_url
+
+    evaluation_config = {
+        "provider": "ANTHROPIC",
+        "model_name": "claude-3-7-sonnet-20250219",
+        "temperature": 0,
+    }
+
+    # Start time for total execution
+    total_start_time = time.time()
+
+    # Run benchmarks based on user selection
+    futures = []
+
+    if args.datasets in ["simpleqa", "both"]:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            simpleqa_future = executor.submit(
+                run_simpleqa_benchmark_wrapper,
+                (
+                    args.examples,
+                    output_dir,
+                    args.resume_from,
+                    search_config,
+                    evaluation_config,
+                ),
+            )
+            futures.append(("simpleqa", simpleqa_future))
+
+    if args.datasets in ["browsecomp", "both"]:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            browsecomp_future = executor.submit(
+                run_browsecomp_benchmark_wrapper,
+                (
+                    args.examples,
+                    output_dir,
+                    args.resume_from,
+                    search_config,
+                    evaluation_config,
+                ),
+            )
+            futures.append(("browsecomp", browsecomp_future))
+
+    # Get results from completed futures
+    simpleqa_results = None
+    browsecomp_results = None
+
+    for dataset_name, future in futures:
+        try:
+            result = future.result()
+            if dataset_name == "simpleqa":
+                simpleqa_results = result
+                print(
+                    f"SimpleQA benchmark completed: {result['new_results']} new, {result['reused_results']} reused"
+                )
+            elif dataset_name == "browsecomp":
+                browsecomp_results = result
+                print(
+                    f"BrowseComp benchmark completed: {result['new_results']} new, {result['reused_results']} reused"
+                )
+        except Exception:
+            logger.exception("Error in benchmark")
+
+    # Calculate total time
+    total_duration = time.time() - total_start_time
+
+    # Print summary
+    print("\n" + "=" * 50)
+    print(" PARALLEL BENCHMARK SUMMARY ")
+    print("=" * 50)
+    print(f"Total duration: {total_duration:.1f} seconds")
+    print(f"Examples per benchmark: {args.examples}")
+    if args.resume_from:
+        print(f"Resumed from: {args.resume_from}")
+
+    if simpleqa_results:
+        print("\nSimpleQA:")
+        print(f"  - Accuracy: {simpleqa_results.get('accuracy', 'N/A')}")
+        print(f"  - New results: {simpleqa_results['new_results']}")
+        print(f"  - Reused results: {simpleqa_results['reused_results']}")
+        print(f"  - Errors: {simpleqa_results.get('errors', 0)}")
+    else:
+        print("\nSimpleQA: Failed or no results")
+
+    if browsecomp_results:
+        print("\nBrowseComp:")
+        print(f"  - Accuracy: {browsecomp_results.get('accuracy', 'N/A')}")
+        print(f"  - New results: {browsecomp_results['new_results']}")
+        print(f"  - Reused results: {browsecomp_results['reused_results']}")
+        print(f"  - Errors: {browsecomp_results.get('errors', 0)}")
+    else:
+        print("\nBrowseComp: Failed or no results")
+
+    print(f"\nResults saved to: {output_dir}")
+    print("=" * 50)
+
+    # Save summary
+    try:
+        summary = {
+            "timestamp": timestamp,
+            "examples_per_benchmark": args.examples,
+            "total_duration": total_duration,
+            "resumed_from": args.resume_from,
+            "simpleqa": simpleqa_results,
+            "browsecomp": browsecomp_results,
+            "model": args.model,
+            "provider": args.provider,
+        }
+
+        with open(
+            Path(output_dir) / "parallel_benchmark_summary.json",
+            "w",
+            encoding="utf-8",
+        ) as f:
+            json.dump(summary, f, indent=2)
+
+    except Exception:
+        logger.exception("Error saving summary")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/run_simpleqa.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+"""
+SimpleQA Benchmark Runner for Local Deep Research.
+
+This script provides a convenient way to run the SimpleQA benchmark.
+
+Usage:
+    # Install dependencies with PDM
+    cd /path/to/local-deep-research
+    pdm install
+
+    # Run the script with PDM
+    pdm run python examples/benchmarks/run_simpleqa.py --help
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+# Import the benchmark functionality
+from local_deep_research.benchmarks.benchmark_functions import evaluate_simpleqa
+
+
+def main():
+    """Run the SimpleQA benchmark with the specified parameters."""
+    parser = argparse.ArgumentParser(description="Run SimpleQA benchmark")
+    parser.add_argument(
+        "--examples", type=int, default=10, help="Number of examples to run"
+    )
+    parser.add_argument(
+        "--iterations", type=int, default=3, help="Number of search iterations"
+    )
+    parser.add_argument(
+        "--questions", type=int, default=3, help="Questions per iteration"
+    )
+    parser.add_argument(
+        "--search-tool", type=str, default="searxng", help="Search tool to use"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path("examples") / "benchmarks" / "results" / "simpleqa"),
+        help="Output directory",
+    )
+    parser.add_argument(
+        "--no-eval", action="store_true", help="Skip evaluation"
+    )
+
+    # Optional evaluation parameters
+    parser.add_argument(
+        "--human-eval", action="store_true", help="Use human evaluation"
+    )
+    parser.add_argument(
+        "--eval-model", type=str, help="Model to use for evaluation"
+    )
+    parser.add_argument(
+        "--eval-provider", type=str, help="Provider to use for evaluation"
+    )
+
+    # Add model configuration options
+    parser.add_argument(
+        "--search-model", type=str, help="Model to use for the search system"
+    )
+    parser.add_argument(
+        "--search-provider",
+        type=str,
+        help="Provider to use for the search system",
+    )
+    parser.add_argument(
+        "--endpoint-url",
+        type=str,
+        help="Endpoint URL for OpenRouter or other API services",
+    )
+    parser.add_argument(
+        "--search-strategy",
+        type=str,
+        default="source_based",
+        choices=[
+            "source_based",
+            "standard",
+            "rapid",
+            "parallel",
+            "iterdrag",
+            "modular",
+        ],
+        help="Search strategy to use (default: source_based)",
+    )
+    parser.add_argument("--api-key", type=str, help="API key for LLM provider")
+
+    args = parser.parse_args()
+
+    print(f"Starting SimpleQA benchmark with {args.examples} examples...")
+
+    # Run the benchmark
+    results = evaluate_simpleqa(
+        num_examples=args.examples,
+        search_iterations=args.iterations,
+        questions_per_iteration=args.questions,
+        search_tool=args.search_tool,
+        human_evaluation=args.human_eval,
+        evaluation_model=args.eval_model,
+        evaluation_provider=args.eval_provider,
+        output_dir=args.output_dir,
+    )
+
+    # Print summary
+    print("\nSimpleQA Benchmark Results:")
+    print(f"  Accuracy: {results.get('accuracy', 0):.3f}")
+    print(f"  Total examples: {results.get('total_examples', 0)}")
+    print(f"  Report saved to: {results.get('report_path', '')}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/scripts/run_benchmark_with_claude_grading.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python
+"""
+Benchmark with Claude API Grading Integration
+
+This script runs a comprehensive evaluation of search strategies with
+proper Claude API integration for grading benchmark results.
+
+Features:
+- Uses the local database for API keys
+- Configures Claude 3 Sonnet for grading
+- Supports SimpleQA and BrowseComp evaluations
+- Provides detailed metrics and accuracy reports
+"""
+
+import os
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+
+
+# Set up Python path
+src_dir = str((Path(__file__).parent / "src").resolve())
+if src_dir not in sys.path:
+    sys.path.insert(0, src_dir)
+
+# Note: Database configuration is now per-user
+# For benchmarks, API keys should be provided via environment variables
+# or configuration files rather than relying on a shared database
+
+# Logger is already imported from loguru
+
+
+def setup_grading_config():
+    """
+    Create a custom evaluation configuration that uses environment variables
+    for API keys and specifically uses Claude 3 Sonnet for grading.
+
+    Returns:
+        Dict containing the evaluation configuration
+    """
+    # Create config that uses Claude 3 Sonnet via Anthropic directly
+    # Only use parameters that get_llm() accepts
+    evaluation_config = {
+        "model_name": "claude-3-sonnet-20240229",  # Correct Anthropic model name
+        "provider": "anthropic",  # Use Anthropic directly
+        "temperature": 0,  # Zero temp for consistent evaluation
+    }
+
+    # Check if anthropic API key is available in environment
+    anthropic_key = os.environ.get("ANTHROPIC_API_KEY")
+    if anthropic_key:
+        print(
+            "Found Anthropic API key in environment, will use Claude 3 Sonnet for grading"
+        )
+    else:
+        print(
+            "Warning: No Anthropic API key found in ANTHROPIC_API_KEY environment variable"
+        )
+        print("Checking for alternative providers...")
+
+        # Try OpenRouter as a fallback
+        openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+        if openrouter_key:
+            print(
+                "Found OpenRouter API key, will use OpenRouter with Claude 3 Sonnet"
+            )
+            evaluation_config = {
+                "model_name": "anthropic/claude-3-sonnet-20240229",  # OpenRouter format
+                "provider": "openai_endpoint",
+                "openai_endpoint_url": "https://openrouter.ai/api/v1",
+                "temperature": 0,
+            }
+        else:
+            print("ERROR: No API keys found in environment variables")
+            print("Please set either ANTHROPIC_API_KEY or OPENROUTER_API_KEY")
+            return None
+
+    return evaluation_config
+
+
+def run_benchmark(strategy="source_based", iterations=1, examples=5):
+    """
+    Run a comprehensive benchmark evaluation of a specific strategy configuration.
+
+    Args:
+        strategy: Search strategy to evaluate (default: source_based)
+        iterations: Number of iterations for the strategy (default: 1)
+        examples: Number of examples to evaluate (default: 5)
+    """
+    # Import the benchmark components
+    try:
+        from local_deep_research.benchmarks.evaluators.browsecomp import (
+            BrowseCompEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.composite import (
+            CompositeBenchmarkEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.simpleqa import (
+            SimpleQAEvaluator,
+        )
+        from local_deep_research.config.llm_config import get_llm
+    except ImportError as e:
+        print(f"Error importing benchmark components: {e}")
+        print("Current sys.path:", sys.path)
+        return
+
+    # Set up custom grading configuration
+    evaluation_config = setup_grading_config()
+    if not evaluation_config:
+        print(
+            "Failed to setup evaluation configuration, proceeding with default config"
+        )
+
+    # Patch the graders module to use our local get_llm
+    try:
+        # This ensures we use the local get_llm function that accesses the database
+        import local_deep_research.benchmarks.graders as graders
+
+        # Store the original function for reference
+        original_get_evaluation_llm = graders.get_evaluation_llm
+
+        # Define a new function that uses our local get_llm directly
+        def custom_get_evaluation_llm(custom_config=None):
+            """
+            Override that uses the local get_llm with database access.
+            """
+            if custom_config is None:
+                custom_config = evaluation_config
+
+            print(f"Getting evaluation LLM with config: {custom_config}")
+            return get_llm(**custom_config)
+
+        # Replace the function with our custom version
+        graders.get_evaluation_llm = custom_get_evaluation_llm
+        print(
+            "Successfully patched graders.get_evaluation_llm to use local get_llm function"
+        )
+
+    except Exception as e:
+        print(f"Error patching graders module: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Create timestamp for output
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    output_dir = str(Path("benchmark_results") / f"claude_grading_{timestamp}")
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+    config = {
+        "search_strategy": strategy,
+        "iterations": iterations,
+        # Add other fixed parameters to ensure a complete run
+        "questions_per_iteration": 1,
+        "max_results": 10,
+        "search_tool": "searxng",  # Specify SearXNG search engine
+        "timeout": 10,  # Very short timeout to speed up the demo
+    }
+
+    # Run SimpleQA benchmark
+    print(
+        f"\n=== Running SimpleQA benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    simpleqa_start = time.time()
+
+    try:
+        # Create SimpleQA evaluator (without the evaluation_config parameter)
+        simpleqa = SimpleQAEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        simpleqa_results = simpleqa.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "simpleqa"),
+        )
+
+        simpleqa_duration = time.time() - simpleqa_start
+        print(
+            f"SimpleQA evaluation complete in {simpleqa_duration:.1f} seconds"
+        )
+        print(f"SimpleQA accuracy: {simpleqa_results.get('accuracy', 0):.4f}")
+        print(f"SimpleQA metrics: {simpleqa_results.get('metrics', {})}")
+
+        # Save results
+        import json
+
+        with open(
+            Path(output_dir) / "simpleqa_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(simpleqa_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during SimpleQA evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run BrowseComp benchmark
+    print(
+        f"\n=== Running BrowseComp benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    browsecomp_start = time.time()
+
+    try:
+        # Create BrowseComp evaluator (without the evaluation_config parameter)
+        browsecomp = BrowseCompEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        browsecomp_results = browsecomp.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "browsecomp"),
+        )
+
+        browsecomp_duration = time.time() - browsecomp_start
+        print(
+            f"BrowseComp evaluation complete in {browsecomp_duration:.1f} seconds"
+        )
+        print(f"BrowseComp score: {browsecomp_results.get('score', 0):.4f}")
+        print(f"BrowseComp metrics: {browsecomp_results.get('metrics', {})}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "browsecomp_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(browsecomp_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during BrowseComp evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run composite benchmark
+    print(
+        f"\n=== Running Composite benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    composite_start = time.time()
+
+    try:
+        # Create composite evaluator with benchmark weights (without evaluation_config parameter)
+        benchmark_weights = {"simpleqa": 0.5, "browsecomp": 0.5}
+        composite = CompositeBenchmarkEvaluator(
+            benchmark_weights=benchmark_weights
+        )
+        composite_results = composite.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "composite"),
+        )
+
+        composite_duration = time.time() - composite_start
+        print(
+            f"Composite evaluation complete in {composite_duration:.1f} seconds"
+        )
+        print(f"Composite score: {composite_results.get('score', 0):.4f}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "composite_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(composite_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during composite evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Generate summary
+    print("\n=== Evaluation Summary ===")
+    print(f"Strategy: {strategy}")
+    print(f"Iterations: {iterations}")
+    print(f"Examples: {examples}")
+    print(f"Results saved to: {output_dir}")
+
+    # If we patched the graders module, restore the original function
+    if "original_get_evaluation_llm" in locals():
+        graders.get_evaluation_llm = original_get_evaluation_llm
+        print("Restored original graders.get_evaluation_llm function")
+
+    return {
+        "simpleqa": simpleqa_results
+        if "simpleqa_results" in locals()
+        else None,
+        "browsecomp": browsecomp_results
+        if "browsecomp_results" in locals()
+        else None,
+        "composite": composite_results
+        if "composite_results" in locals()
+        else None,
+    }
+
+
+def main():
+    # Parse command line arguments
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run benchmark with Claude API grading"
+    )
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="source_based",
+        help="Strategy to evaluate (default: source_based)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=1,
+        help="Number of iterations (default: 1)",
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=5,
+        help="Number of examples to evaluate (default: 5)",
+    )
+
+    args = parser.parse_args()
+
+    print(
+        f"Starting benchmark of {args.strategy} strategy with {args.iterations} iterations"
+    )
+    print(f"Evaluating with {args.examples} examples")
+
+    # Run the evaluation
+    results = run_benchmark(
+        strategy=args.strategy,
+        iterations=args.iterations,
+        examples=args.examples,
+    )
+
+    # Return success if at least one benchmark completed
+    return 0 if any(results.values()) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/scripts/run_focused_benchmark_fixed.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python
+"""
+Focused source-based strategy evaluation with complete metrics.
+
+This script runs a focused evaluation of the source-based strategy with
+comprehensive metrics for both SimpleQA and BrowseComp benchmarks.
+
+Updated version that properly uses the local get_llm function for grading,
+accesses the database for API keys, and uses Claude Anthropic 3.7 for grading.
+"""
+
+import os
+import sys
+import time
+from datetime import datetime, UTC
+from pathlib import Path
+
+
+# Set up Python path
+src_dir = str((Path(__file__).parent / "src").resolve())
+if src_dir not in sys.path:
+    sys.path.insert(0, src_dir)
+
+# Use environment variables for configuration
+# The system should be configured with proper environment variables:
+# - ANTHROPIC_API_KEY for Anthropic API access
+# - OPENROUTER_API_KEY for OpenRouter API access (if used)
+# - LDR_DATA_DIR for data directory location (if needed)
+data_dir = os.environ.get("LDR_DATA_DIR", str(Path(src_dir) / "data"))
+
+
+def setup_grading_config():
+    """
+    Create a custom evaluation configuration that uses environment variables
+    for API keys and specifically uses Claude Anthropic 3.7 Sonnet for grading.
+
+    Returns:
+        Dict containing the evaluation configuration
+    """
+    # No need to import database utilities anymore
+
+    # Create config that uses Claude 3 Sonnet via Anthropic directly
+    # This will use the API key from environment variables
+    # Only use parameters that get_llm() accepts
+    evaluation_config = {
+        "model_name": "claude-3-sonnet-20240229",  # Correct Anthropic model name
+        "provider": "anthropic",  # Use Anthropic directly
+        "temperature": 0,  # Zero temp for consistent evaluation
+    }
+
+    # Check if anthropic API key is available in environment
+    anthropic_key = os.environ.get("ANTHROPIC_API_KEY")
+    if anthropic_key:
+        print(
+            "Found Anthropic API key in environment, will use Claude 3.7 Sonnet for grading"
+        )
+    else:
+        print("Warning: No Anthropic API key found in environment")
+        print("Checking for alternative providers...")
+
+        # Try OpenRouter as a fallback
+        openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+        if openrouter_key:
+            print(
+                "Found OpenRouter API key, will use OpenRouter with Claude 3.7 Sonnet"
+            )
+            evaluation_config = {
+                "model_name": "anthropic/claude-3-7-sonnet",  # OpenRouter format
+                "provider": "openai_endpoint",
+                "openai_endpoint_url": "https://openrouter.ai/api/v1",
+                "temperature": 0,
+            }
+
+    return evaluation_config
+
+
+def run_direct_evaluation(strategy="source_based", iterations=1, examples=5):
+    """
+    Run direct evaluation of a specific strategy configuration.
+
+    Args:
+        strategy: Search strategy to evaluate (default: source_based)
+        iterations: Number of iterations for the strategy (default: 1)
+        examples: Number of examples to evaluate (default: 5)
+    """
+    # Import the benchmark components
+    try:
+        from local_deep_research.benchmarks.evaluators.browsecomp import (
+            BrowseCompEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.composite import (
+            CompositeBenchmarkEvaluator,
+        )
+        from local_deep_research.benchmarks.evaluators.simpleqa import (
+            SimpleQAEvaluator,
+        )
+        from local_deep_research.config.llm_config import get_llm
+    except ImportError as e:
+        print(f"Error importing benchmark components: {e}")
+        print("Current sys.path:", sys.path)
+        return
+
+    # Set up custom grading configuration
+    evaluation_config = setup_grading_config()
+    if not evaluation_config:
+        print(
+            "Failed to setup evaluation configuration, proceeding with default config"
+        )
+
+    # Patch the graders module to use our local get_llm
+    try:
+        # This ensures we use the local get_llm function that accesses the database
+        import local_deep_research.benchmarks.graders as graders
+
+        # Store the original function for reference
+        original_get_evaluation_llm = graders.get_evaluation_llm
+
+        # Define a new function that uses our local get_llm directly
+        def custom_get_evaluation_llm(custom_config=None):
+            """
+            Override that uses the local get_llm with database access.
+            """
+            if custom_config is None:
+                custom_config = evaluation_config
+
+            print(f"Getting evaluation LLM with config: {custom_config}")
+            return get_llm(**custom_config)
+
+        # Replace the function with our custom version
+        graders.get_evaluation_llm = custom_get_evaluation_llm
+        print(
+            "Successfully patched graders.get_evaluation_llm to use local get_llm function"
+        )
+
+    except Exception as e:
+        print(f"Error patching graders module: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Create timestamp for output
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    output_dir = str(Path("benchmark_results") / f"direct_eval_{timestamp}")
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+    config = {
+        "search_strategy": strategy,
+        "iterations": iterations,
+        # Add other fixed parameters to ensure a complete run
+        "questions_per_iteration": 1,
+        "max_results": 10,
+        "search_tool": "searxng",  # Specify SearXNG search engine
+        "timeout": 10,  # Very short timeout to speed up the demo
+    }
+
+    # Run SimpleQA benchmark
+    print(
+        f"\n=== Running SimpleQA benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    simpleqa_start = time.time()
+
+    try:
+        # Create SimpleQA evaluator (without the evaluation_config parameter)
+        simpleqa = SimpleQAEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        simpleqa_results = simpleqa.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "simpleqa"),
+        )
+
+        simpleqa_duration = time.time() - simpleqa_start
+        print(
+            f"SimpleQA evaluation complete in {simpleqa_duration:.1f} seconds"
+        )
+        print(f"SimpleQA accuracy: {simpleqa_results.get('accuracy', 0):.4f}")
+        print(f"SimpleQA metrics: {simpleqa_results.get('metrics', {})}")
+
+        # Save results
+        import json
+
+        with open(
+            Path(output_dir) / "simpleqa_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(simpleqa_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during SimpleQA evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run BrowseComp benchmark
+    print(
+        f"\n=== Running BrowseComp benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    browsecomp_start = time.time()
+
+    try:
+        # Create BrowseComp evaluator (without the evaluation_config parameter)
+        browsecomp = BrowseCompEvaluator()
+
+        # The evaluation_config will be used automatically through our patched function
+        # when grade_results is called inside the evaluator
+        browsecomp_results = browsecomp.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "browsecomp"),
+        )
+
+        browsecomp_duration = time.time() - browsecomp_start
+        print(
+            f"BrowseComp evaluation complete in {browsecomp_duration:.1f} seconds"
+        )
+        print(f"BrowseComp score: {browsecomp_results.get('score', 0):.4f}")
+        print(f"BrowseComp metrics: {browsecomp_results.get('metrics', {})}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "browsecomp_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(browsecomp_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during BrowseComp evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Run composite benchmark
+    print(
+        f"\n=== Running Composite benchmark with {strategy} strategy, {iterations} iterations ==="
+    )
+    composite_start = time.time()
+
+    try:
+        # Create composite evaluator with benchmark weights (without evaluation_config parameter)
+        benchmark_weights = {"simpleqa": 0.5, "browsecomp": 0.5}
+        composite = CompositeBenchmarkEvaluator(
+            benchmark_weights=benchmark_weights
+        )
+        composite_results = composite.evaluate(
+            config,
+            num_examples=examples,
+            output_dir=str(Path(output_dir) / "composite"),
+        )
+
+        composite_duration = time.time() - composite_start
+        print(
+            f"Composite evaluation complete in {composite_duration:.1f} seconds"
+        )
+        print(f"Composite score: {composite_results.get('score', 0):.4f}")
+
+        # Save results
+        with open(
+            Path(output_dir) / "composite_results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(composite_results, f, indent=2)
+    except Exception as e:
+        print(f"Error during composite evaluation: {e}")
+        import traceback
+
+        traceback.print_exc()
+
+    # Generate summary
+    print("\n=== Evaluation Summary ===")
+    print(f"Strategy: {strategy}")
+    print(f"Iterations: {iterations}")
+    print(f"Examples: {examples}")
+    print(f"Results saved to: {output_dir}")
+
+    # If we patched the graders module, restore the original function
+    if "original_get_evaluation_llm" in locals():
+        graders.get_evaluation_llm = original_get_evaluation_llm
+        print("Restored original graders.get_evaluation_llm function")
+
+    return {
+        "simpleqa": simpleqa_results
+        if "simpleqa_results" in locals()
+        else None,
+        "browsecomp": browsecomp_results
+        if "browsecomp_results" in locals()
+        else None,
+        "composite": composite_results
+        if "composite_results" in locals()
+        else None,
+    }
+
+
+def main():
+    # Parse command line arguments
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run focused strategy benchmark"
+    )
+    parser.add_argument(
+        "--strategy",
+        type=str,
+        default="source_based",
+        help="Strategy to evaluate (default: source_based)",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=1,
+        help="Number of iterations (default: 1)",
+    )
+    parser.add_argument(
+        "--examples",
+        type=int,
+        default=5,
+        help="Number of examples to evaluate (default: 5)",
+    )
+
+    args = parser.parse_args()
+
+    print(
+        f"Starting focused evaluation of {args.strategy} strategy with {args.iterations} iterations"
+    )
+    print(f"Evaluating with {args.examples} examples")
+
+    # Run the evaluation
+    results = run_direct_evaluation(
+        strategy=args.strategy,
+        iterations=args.iterations,
+        examples=args.examples,
+    )
+
+    # Return success if at least one benchmark completed
+    return 0 if any(results.values()) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/benchmarks/scripts/run_grader_only.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+"""
+Run Claude API grading on existing benchmark results.
+
+This script takes existing benchmark results and runs the grading phase
+without re-executing the benchmark itself.
+"""
+
+import argparse
+import os
+import sys
+import time
+from pathlib import Path
+
+
+# Set up Python path
+src_dir = str((Path(__file__).parent / "src").resolve())
+if src_dir not in sys.path:
+    sys.path.insert(0, src_dir)
+
+# Use environment variables for configuration
+# The system should be configured with proper environment variables:
+# - ANTHROPIC_API_KEY for Anthropic API access
+# - OPENROUTER_API_KEY for OpenRouter API access (if used)
+# - LDR_DATA_DIR for data directory location (if needed)
+data_dir = os.environ.get("LDR_DATA_DIR", str(Path(src_dir) / "data"))
+
+
+def setup_grading_config():
+    """
+    Create a custom evaluation configuration that uses environment variables
+    for API keys and specifically uses Claude 3 Sonnet for grading.
+
+    Returns:
+        Dict containing the evaluation configuration
+    """
+    # No need to import database utilities anymore
+
+    # Create config that uses Claude 3 Sonnet via Anthropic directly
+    # This will use the API key from environment variables
+    # Only use parameters that get_llm() accepts
+    evaluation_config = {
+        "model_name": "claude-3-sonnet-20240229",  # Correct Anthropic model name
+        "provider": "anthropic",  # Use Anthropic directly
+        "temperature": 0,  # Zero temp for consistent evaluation
+    }
+
+    # Check if anthropic API key is available in environment
+    anthropic_key = os.environ.get("ANTHROPIC_API_KEY")
+    if anthropic_key:
+        print(
+            "Found Anthropic API key in environment, will use Claude 3 Sonnet for grading"
+        )
+    else:
+        print("Warning: No Anthropic API key found in environment")
+        print("Checking for alternative providers...")
+
+        # Try OpenRouter as a fallback
+        openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+        if openrouter_key:
+            print(
+                "Found OpenRouter API key, will use OpenRouter with Claude 3 Sonnet"
+            )
+            evaluation_config = {
+                "model_name": "anthropic/claude-3-sonnet-20240229",  # OpenRouter format
+                "provider": "openai_endpoint",
+                "openai_endpoint_url": "https://openrouter.ai/api/v1",
+                "temperature": 0,
+            }
+
+    return evaluation_config
+
+
+def grade_benchmark_results(results_path, dataset_type="simpleqa"):
+    """
+    Grade benchmark results using Claude API.
+
+    Args:
+        results_path: Path to the results JSONL file
+        dataset_type: Type of dataset (simpleqa or browsecomp)
+
+    Returns:
+        Path to the evaluation file
+    """
+    try:
+        # Import grading components
+        from local_deep_research.benchmarks.graders import grade_results
+        from local_deep_research.config.llm_config import get_llm
+
+        # Set up custom grading configuration
+        evaluation_config = setup_grading_config()
+        if not evaluation_config:
+            print(
+                "Failed to setup evaluation configuration, proceeding with default config"
+            )
+
+        # Patch the graders module to use our local get_llm
+        try:
+            # This ensures we use the local get_llm function that accesses the database
+            import local_deep_research.benchmarks.graders as graders
+
+            # Store the original function for reference
+            original_get_evaluation_llm = graders.get_evaluation_llm
+
+            # Define a new function that uses our local get_llm directly
+            def custom_get_evaluation_llm(custom_config=None):
+                """
+                Override that uses the local get_llm with database access.
+                """
+                if custom_config is None:
+                    custom_config = evaluation_config
+
+                print(f"Getting evaluation LLM with config: {custom_config}")
+                return get_llm(**custom_config)
+
+            # Replace the function with our custom version
+            graders.get_evaluation_llm = custom_get_evaluation_llm
+            print(
+                "Successfully patched graders.get_evaluation_llm to use local get_llm function"
+            )
+
+        except Exception as e:
+            print(f"Error patching graders module: {e}")
+            import traceback
+
+            traceback.print_exc()
+
+        # Create the evaluation output path
+        results_dir = str(Path(results_path).parent)
+        results_filename = Path(results_path).name
+        evaluation_filename = results_filename.replace(
+            "_results.jsonl", "_evaluation.jsonl"
+        )
+        evaluation_path = str(Path(results_dir) / evaluation_filename)
+
+        # Run the grading
+        print("Starting grading of benchmark results...")
+        grading_start_time = time.time()
+        try:
+            evaluation_results = grade_results(
+                results_file=results_path,
+                output_file=evaluation_path,
+                dataset_type=dataset_type,
+                evaluation_config=evaluation_config,
+                progress_callback=lambda current, total, meta: print(
+                    f"Grading progress: {current + 1}/{total} ({((current + 1) / total * 100):.1f}%)"
+                ),
+            )
+
+            grading_duration = time.time() - grading_start_time
+            accuracy = (
+                sum(1 for r in evaluation_results if r.get("is_correct", False))
+                / len(evaluation_results)
+                if evaluation_results
+                else 0
+            )
+
+            print(f"\nGrading complete in {grading_duration:.1f} seconds")
+            print(f"Accuracy: {accuracy:.4f}")
+            print(f"Graded {len(evaluation_results)} examples")
+            print(f"Results saved to: {evaluation_path}")
+
+            # If we patched the graders module, restore the original function
+            if "original_get_evaluation_llm" in locals():
+                graders.get_evaluation_llm = original_get_evaluation_llm
+                print("Restored original graders.get_evaluation_llm function")
+
+            return evaluation_path
+
+        except Exception as e:
+            print(f"Error during grading: {e}")
+            import traceback
+
+            traceback.print_exc()
+            return None
+
+    except ImportError as e:
+        print(f"Error importing benchmark components: {e}")
+        print("Current sys.path:", sys.path)
+        return None
+
+
+def generate_summary(evaluation_path, output_dir=None):
+    """
+    Generate a summary report of the evaluation results.
+
+    Args:
+        evaluation_path: Path to the evaluation JSONL file
+        output_dir: Directory to save the summary report
+
+    Returns:
+        Path to the summary report
+    """
+    try:
+        import json
+
+        from local_deep_research.benchmarks.metrics import (
+            calculate_metrics,
+            generate_report,
+        )
+
+        # Load evaluation results
+        evaluation_results = []
+        with open(evaluation_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    evaluation_results.append(json.loads(line))
+
+        # Calculate metrics
+        metrics = calculate_metrics(evaluation_results)
+
+        # Determine output directory
+        if output_dir is None:
+            output_dir = str(Path(evaluation_path).parent)
+
+        # Generate report
+        report_path = str(Path(output_dir) / "evaluation_report.md")
+        generate_report(
+            metrics=metrics,
+            output_file=report_path,
+            dataset_type="simpleqa"
+            if "simpleqa" in evaluation_path
+            else "browsecomp",
+        )
+
+        # Print summary
+        print("\nEvaluation Summary:")
+        print(f"Total examples: {metrics['total_examples']}")
+        print(f"Correct: {metrics['correct']}")
+        print(f"Accuracy: {metrics['accuracy']:.4f}")
+        print(
+            f"Average processing time: {metrics['average_processing_time']:.2f} seconds"
+        )
+        print(f"Summary report saved to: {report_path}")
+
+        return report_path
+
+    except Exception as e:
+        print(f"Error generating summary: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return None
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Run Claude API grading on existing benchmark results"
+    )
+    parser.add_argument(
+        "--results",
+        type=str,
+        required=True,
+        help="Path to the results JSONL file",
+    )
+    parser.add_argument(
+        "--dataset-type",
+        type=str,
+        default="simpleqa",
+        choices=["simpleqa", "browsecomp"],
+        help="Type of dataset (simpleqa or browsecomp)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=None,
+        help="Directory to save output files. If not specified, uses the directory of the results file.",
+    )
+
+    args = parser.parse_args()
+
+    # Check if the results file exists
+    if not Path(args.results).exists():
+        print(f"Error: Results file not found: {args.results}")
+        return 1
+
+    # Run grading
+    start_time = time.time()
+    print(
+        f"Starting grading of {args.dataset_type} benchmark results from: {args.results}"
+    )
+
+    evaluation_path = grade_benchmark_results(args.results, args.dataset_type)
+    if not evaluation_path:
+        print("Grading failed")
+        return 1
+
+    # Generate summary
+    report_path = generate_summary(evaluation_path, args.output_dir)
+    if not report_path:
+        print("Summary generation failed")
+        return 1
+
+    # Print overall timing
+    total_time = time.time() - start_time
+    print(f"\nTotal processing time: {total_time:.1f} seconds")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())