chore: import upstream snapshot with attribution

python/packages/lab/gaia/samples/azure_ai_agent.py

@@ -0,0 +1,67 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Azure AI Agent factory for GAIA benchmark.
+
+This module provides a factory function to create an Azure AI agent
+configured for GAIA benchmark tasks.
+
+Required Environment Variables:
+    FOUNDRY_PROJECT_ENDPOINT: Azure AI project endpoint URL
+    FOUNDRY_MODEL: Name of the model deployment to use
+
+Optional Environment Variables:
+    BING_CONNECTION_ID: ID of the Bing connection for web search
+
+Authentication:
+    Uses Azure CLI credentials via AzureCliCredential.
+    Run `az login` before executing to authenticate.
+
+Example:
+    export FOUNDRY_PROJECT_ENDPOINT="https://your-project.azure.com"
+    export FOUNDRY_MODEL="gpt-4o"
+    export BING_CONNECTION_ID="connection-id"
+    az login
+"""
+
+import os
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from agent_framework import Agent
+from agent_framework.foundry import FoundryChatClient
+from azure.identity.aio import AzureCliCredential
+
+
+@asynccontextmanager
+async def create_gaia_agent() -> AsyncIterator[Agent]:
+    """Create an Azure AI agent configured for GAIA benchmark tasks.
+
+    The agent is configured with:
+    - Bing Search tool for web information retrieval
+    - Code Interpreter tool for calculations and data analysis
+
+    Yields:
+        Agent: A configured agent ready to run GAIA tasks.
+
+    Example:
+        async with create_gaia_agent() as agent:
+            result = await agent.run("What is the capital of France?")
+            print(result.text)
+    """
+    async with (
+        AzureCliCredential() as credential,
+        FoundryChatClient(
+            project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
+            model=os.environ["FOUNDRY_MODEL"],
+            credential=credential,
+        ).as_agent(
+            name="GaiaAgent",
+            instructions="Solve tasks to your best ability. Use Bing Search to find "
+            "information and Code Interpreter to perform calculations and data analysis.",
+            tools=[
+                FoundryChatClient.get_web_search_tool(),
+                FoundryChatClient.get_code_interpreter_tool(),
+            ],
+        ) as agent,
+    ):
+        yield agent

python/packages/lab/gaia/samples/gaia_sample.py

@@ -0,0 +1,295 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""GAIA Benchmark Sample.
+
+Run the GAIA (General AI Assistant) benchmark with configurable agent providers,
+telemetry options, and benchmark parameters.
+
+Agent Providers:
+    - Azure AI (default): See azure_ai_agent.py for required environment variables
+    - OpenAI: See openai_agent.py for required environment variables
+
+Prerequisites:
+    1. Set HF_TOKEN environment variable with your Hugging Face token:
+       - Get token: https://huggingface.co/settings/tokens
+       - Request dataset access: https://huggingface.co/datasets/gaia-benchmark/GAIA
+       - Set: export HF_TOKEN="your-huggingface-token"
+
+    2. Configure your chosen agent provider (see agent module files for details)
+
+Telemetry:
+    When using --otlp-endpoint or --trace-file, OpenTelemetry will export trace data
+    in JSON format to the console in addition to the configured endpoints. This is
+    expected behavior from the OpenTelemetry SDK and provides visibility into the
+    telemetry being captured. The traces are also exported to:
+    - OTLP endpoint (e.g., Aspire Dashboard) if --otlp-endpoint is specified
+    - Local file if --trace-file is specified
+
+    To suppress console output, redirect stderr: `python gaia_sample.py 2>/dev/null`
+
+Usage:
+    # Run with default settings (Azure AI agent)
+    uv run python gaia_sample.py
+
+    # Run with OpenAI agent
+    uv run python gaia_sample.py --agent-provider openai
+
+    # Run with telemetry export to Aspire Dashboard
+    uv run python gaia_sample.py --otlp-endpoint http://localhost:4318
+
+    # See all options
+    uv run python gaia_sample.py --help
+"""
+
+import argparse
+
+from agent_framework.lab.gaia import GAIA, Evaluation, GAIATelemetryConfig, Prediction, Task
+
+
+async def evaluate_task(task: Task, prediction: Prediction) -> Evaluation:
+    """Evaluate the prediction for a given task."""
+    # Simple evaluation: check if the prediction contains the answer
+    is_correct = (task.answer or "").lower() in prediction.prediction.lower()
+    return Evaluation(is_correct=is_correct, score=1 if is_correct else 0)
+
+
+async def main(
+    otlp_endpoint: str | None = None,
+    trace_file: str | None = None,
+    result_file: str | None = None,
+    data_dir: str | None = None,
+    agent_provider: str = "azure-ai",
+    level: int | list[int] = 1,
+    max_n: int = 2,
+    parallel: int = 1,
+    timeout: int = 120,
+) -> None:
+    """Run GAIA benchmark with telemetry configuration.
+
+    Args:
+        otlp_endpoint: Optional OTLP endpoint URL for exporting traces (e.g., http://localhost:4318)
+        trace_file: Optional file path to export traces to. If None, traces won't be saved to file.
+        result_file: Optional file path to save benchmark results. If None, results won't be saved to file.
+        data_dir: Directory to cache GAIA dataset. If None, uses temp directory.
+        agent_provider: Agent provider to use: 'azure-ai' or 'openai' (default: 'azure-ai')
+        level: GAIA level(s) to run (1, 2, or 3)
+        max_n: Maximum number of tasks to run per level
+        parallel: Number of parallel tasks to run
+        timeout: Timeout per task in seconds
+    """
+    # Check for required Hugging Face token
+    import logging
+    import os
+
+    # Suppress console logging for traces and verbose SDK output
+    logging.getLogger("opentelemetry").setLevel(logging.ERROR)
+    logging.getLogger("azure").setLevel(logging.WARNING)
+    logging.getLogger("agent_framework").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+    # Suppress OpenTelemetry exporters console output
+    import os as _os
+
+    _os.environ.setdefault("OTEL_PYTHON_LOG_LEVEL", "error")
+
+    # Print trace export configuration
+    print("\n=== Telemetry Configuration ===")
+    if trace_file:
+        print(f"📁 Trace file: {os.path.abspath(trace_file)}")
+    else:
+        print("📁 Trace file: disabled")
+
+    if otlp_endpoint:
+        print(f"🌐 OTLP endpoint: {otlp_endpoint}")
+    else:
+        print("🌐 OTLP endpoint: disabled")
+
+    if result_file:
+        print(f"📊 Results file: {os.path.abspath(result_file)}")
+    else:
+        print("📊 Results file: disabled")
+
+    print("\n=== Run Configuration ===")
+    print(f"🤖 Agent provider: {agent_provider}")
+    if data_dir:
+        print(f"📂 Data directory: {os.path.abspath(data_dir)}")
+    else:
+        import tempfile
+        from pathlib import Path
+
+        default_data_dir = Path(tempfile.gettempdir()) / "data_gaia_hub"
+        print(f"📂 Data directory: {default_data_dir} (default)")
+    print(f"🎯 Level: {level}")
+    print(f"🔢 Max tasks: {max_n}")
+    print(f"⚡ Parallel: {parallel}")
+    print(f"⏱️  Timeout: {timeout}s")
+    print()
+
+    # Import the appropriate agent factory based on provider
+    if agent_provider == "azure-ai":
+        from azure_ai_agent import create_gaia_agent
+    elif agent_provider == "openai":
+        from openai_agent import create_gaia_agent
+    else:
+        raise ValueError(f"Unknown agent provider: {agent_provider}. Use 'azure-ai' or 'openai'.")
+
+    # Configure telemetry for tracing
+    telemetry_config = GAIATelemetryConfig(
+        enable_tracing=True,  # Enable OpenTelemetry tracing
+        trace_to_file=trace_file is not None,  # Export traces to local file only if path provided
+        file_path=trace_file,  # Custom file path for traces (can be None)
+        otlp_endpoint=otlp_endpoint,  # Optional OTLP endpoint for Aspire Dashboard or other collectors
+    )
+
+    # Create a single agent once and reuse it for all tasks
+    async with create_gaia_agent() as agent:
+
+        async def run_task(task: Task) -> Prediction:
+            """Run a single GAIA task and return the prediction using the shared agent."""
+            input_message = f"Task: {task.question}"
+            if task.file_name:
+                input_message += f"\nFile: {task.file_name}"
+            result = await agent.run(input_message)
+            return Prediction(prediction=result.text, messages=result.messages)
+
+        # Create the GAIA benchmark runner with telemetry configuration
+        runner = GAIA(
+            evaluator=evaluate_task,
+            telemetry_config=telemetry_config,
+            data_dir=data_dir,
+        )
+
+        # Run the benchmark with the task runner.
+        # By default, this will check for locally cached benchmark data and checkout
+        # the latest version from HuggingFace if not found.
+        # Note: The GAIA dataset has been updated to use Parquet format.
+        # If you encounter issues, try using validation split which has labeled data.
+        results = await runner.run(
+            run_task,
+            level=level,
+            max_n=max_n,
+            parallel=parallel,
+            timeout=timeout,
+            out=result_file,  # Output file to save results including detailed traces (optional, None = no file output)
+        )
+
+    # Print summary similar to the viewer in gaia.py
+    total = len(results)
+    correct = sum(1 for r in results if r.evaluation.is_correct)
+    accuracy = correct / total if total > 0 else 0.0
+    avg_runtime = sum(r.runtime_seconds or 0 for r in results) / total if total > 0 else 0.0
+
+    print("\n=== GAIA Benchmark Summary ===")
+    print(f"📝 Total: {total}, ✅ Correct: {correct}, 🎯 Accuracy: {accuracy:.3f}")
+    print(f"⏱️  Average runtime: {avg_runtime:.2f}s")
+    if result_file:
+        print(f"💾 Detailed results saved to: {result_file}")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description="Run GAIA benchmark with optional telemetry export to OTLP endpoint and/or file",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with default settings
+  python gaia_sample.py
+
+  # Run with custom data directory
+  python gaia_sample.py --data-dir ./gaia_data
+
+  # Run with OpenAI agent provider
+  python gaia_sample.py --agent-provider openai
+
+  # Run with trace file export
+  python gaia_sample.py --trace-file gaia_benchmark_traces.jsonl
+
+  # Run level 2 tasks with 5 maximum tasks
+  python gaia_sample.py --level 2 --max-n 5
+
+  # Run with OTLP export to Aspire Dashboard and custom settings
+  python gaia_sample.py --otlp-endpoint http://localhost:4318 --level 1 --max-n 10 --parallel 2
+
+  # Run with all options configured
+  python gaia_sample.py --agent-provider openai \
+  --trace-file traces.jsonl \
+  --result-file results.jsonl \
+  --otlp-endpoint http://localhost:4318 --level 1 --max-n 5 --parallel 2 --timeout 180
+        """,
+    )
+    parser.add_argument(
+        "--otlp-endpoint",
+        type=str,
+        default=None,
+        help="OTLP endpoint URL for exporting traces (e.g., http://localhost:4318 for Aspire Dashboard)",
+    )
+    parser.add_argument(
+        "--trace-file",
+        type=str,
+        default=None,
+        help="File path to export traces to (e.g., gaia_benchmark_traces.jsonl). "
+        "If not set, traces won't be saved to file.",
+    )
+    parser.add_argument(
+        "--result-file",
+        type=str,
+        default="gaia_results_level1.jsonl",
+        help="File path to save benchmark results (default: gaia_results_level1.jsonl)",
+    )
+    parser.add_argument(
+        "--data-dir",
+        type=str,
+        default=None,
+        help="Directory to cache GAIA dataset. If not set, uses system temp directory.",
+    )
+    parser.add_argument(
+        "--agent-provider",
+        type=str,
+        default="azure-ai",
+        choices=["azure-ai", "openai"],
+        help="Agent provider to use: 'azure-ai' or 'openai' (default: 'azure-ai')",
+    )
+    parser.add_argument(
+        "--level",
+        type=int,
+        default=1,
+        choices=[1, 2, 3],
+        help="GAIA benchmark level to run: 1, 2, or 3 (default: 1)",
+    )
+    parser.add_argument(
+        "--max-n",
+        type=int,
+        default=2,
+        help="Maximum number of tasks to run per level (default: 2)",
+    )
+    parser.add_argument(
+        "--parallel",
+        type=int,
+        default=1,
+        help="Number of parallel tasks to run (default: 1)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=120,
+        help="Timeout per task in seconds (default: 120)",
+    )
+    args = parser.parse_args()
+
+    asyncio.run(
+        main(
+            otlp_endpoint=args.otlp_endpoint,
+            trace_file=args.trace_file,
+            result_file=args.result_file,
+            data_dir=args.data_dir,
+            agent_provider=args.agent_provider,
+            level=args.level,
+            max_n=args.max_n,
+            parallel=args.parallel,
+            timeout=args.timeout,
+        )
+    )

python/packages/lab/gaia/samples/openai_agent.py

@@ -0,0 +1,61 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""OpenAI Agent factory for GAIA benchmark.
+
+This module provides a factory function to create an OpenAI agent
+configured for GAIA benchmark tasks using the OpenAI Responses API.
+
+Required Environment Variables:
+    OPENAI_API_KEY: Your OpenAI API key
+    OPENAI_CHAT_MODEL: Model to use with Responses API (e.g., gpt-4o, gpt-4o-mini)
+
+Optional Environment Variables:
+    OPENAI_BASE_URL: Custom API base URL if using a proxy or compatible service
+    OPENAI_ORG_ID: Organization ID for OpenAI API (if applicable)
+
+Authentication:
+    Uses OPENAI_API_KEY environment variable.
+    Get your API key from: https://platform.openai.com/api-keys
+
+Example:
+    export OPENAI_API_KEY="sk-..."
+    export OPENAI_CHAT_MODEL="gpt-4o"
+"""
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from agent_framework import Agent
+from agent_framework.openai import OpenAIChatClient
+
+
+@asynccontextmanager
+async def create_gaia_agent() -> AsyncIterator[Agent]:
+    """Create an OpenAI agent configured for GAIA benchmark tasks.
+
+    Uses OpenAI Responses API for enhanced capabilities.
+
+    The agent is configured with:
+    - Web Search tool for information retrieval
+    - Code Interpreter tool for calculations and data analysis
+
+    Yields:
+        Agent: A configured agent ready to run GAIA tasks.
+
+    Example:
+        async with create_gaia_agent() as agent:
+            result = await agent.run("What is the capital of France?")
+            print(result.text)
+    """
+    client = OpenAIChatClient()
+
+    async with client.as_agent(
+        name="GaiaAgent",
+        instructions="Solve tasks to your best ability. Use Web Search to find "
+        "information and Code Interpreter to perform calculations and data analysis.",
+        tools=[
+            OpenAIChatClient.get_web_search_tool(),
+            OpenAIChatClient.get_code_interpreter_tool(),
+        ],
+    ) as agent:
+        yield agent