chore: import upstream snapshot with attribution

examples/benchmarks/run_simpleqa.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+"""
+SimpleQA Benchmark Runner for Local Deep Research.
+
+This script provides a convenient way to run the SimpleQA benchmark.
+
+Usage:
+    # Install dependencies with PDM
+    cd /path/to/local-deep-research
+    pdm install
+
+    # Run the script with PDM
+    pdm run python examples/benchmarks/run_simpleqa.py --help
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+# Import the benchmark functionality
+from local_deep_research.benchmarks.benchmark_functions import evaluate_simpleqa
+
+
+def main():
+    """Run the SimpleQA benchmark with the specified parameters."""
+    parser = argparse.ArgumentParser(description="Run SimpleQA benchmark")
+    parser.add_argument(
+        "--examples", type=int, default=10, help="Number of examples to run"
+    )
+    parser.add_argument(
+        "--iterations", type=int, default=3, help="Number of search iterations"
+    )
+    parser.add_argument(
+        "--questions", type=int, default=3, help="Questions per iteration"
+    )
+    parser.add_argument(
+        "--search-tool", type=str, default="searxng", help="Search tool to use"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path("examples") / "benchmarks" / "results" / "simpleqa"),
+        help="Output directory",
+    )
+    parser.add_argument(
+        "--no-eval", action="store_true", help="Skip evaluation"
+    )
+
+    # Optional evaluation parameters
+    parser.add_argument(
+        "--human-eval", action="store_true", help="Use human evaluation"
+    )
+    parser.add_argument(
+        "--eval-model", type=str, help="Model to use for evaluation"
+    )
+    parser.add_argument(
+        "--eval-provider", type=str, help="Provider to use for evaluation"
+    )
+
+    # Add model configuration options
+    parser.add_argument(
+        "--search-model", type=str, help="Model to use for the search system"
+    )
+    parser.add_argument(
+        "--search-provider",
+        type=str,
+        help="Provider to use for the search system",
+    )
+    parser.add_argument(
+        "--endpoint-url",
+        type=str,
+        help="Endpoint URL for OpenRouter or other API services",
+    )
+    parser.add_argument(
+        "--search-strategy",
+        type=str,
+        default="source_based",
+        choices=[
+            "source_based",
+            "standard",
+            "rapid",
+            "parallel",
+            "iterdrag",
+            "modular",
+        ],
+        help="Search strategy to use (default: source_based)",
+    )
+    parser.add_argument("--api-key", type=str, help="API key for LLM provider")
+
+    args = parser.parse_args()
+
+    print(f"Starting SimpleQA benchmark with {args.examples} examples...")
+
+    # Run the benchmark
+    results = evaluate_simpleqa(
+        num_examples=args.examples,
+        search_iterations=args.iterations,
+        questions_per_iteration=args.questions,
+        search_tool=args.search_tool,
+        human_evaluation=args.human_eval,
+        evaluation_model=args.eval_model,
+        evaluation_provider=args.eval_provider,
+        output_dir=args.output_dir,
+    )
+
+    # Print summary
+    print("\nSimpleQA Benchmark Results:")
+    print(f"  Accuracy: {results.get('accuracy', 0):.3f}")
+    print(f"  Total examples: {results.get('total_examples', 0)}")
+    print(f"  Report saved to: {results.get('report_path', '')}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())