chore: import upstream snapshot with attribution

examples/integration-pydantic-ai/README.md

@@ -0,0 +1,35 @@
+# integration-pydantic-ai (Pydantic AI Integration)
+
+This example demonstrates how to evaluate [PydanticAI](https://ai.pydantic.dev/) agents using promptfoo. PydanticAI is a Python agent framework that provides structured outputs and type safety for AI applications.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example integration-pydantic-ai
+cd integration-pydantic-ai
+```
+
+## Quick Start
+
+```bash
+cd integration-pydantic-ai
+pip install -r requirements.txt
+export OPENAI_API_KEY=your_openai_api_key_here
+npx promptfoo@latest eval
+npx promptfoo@latest view
+```
+
+## What This Shows
+
+- Creating a PydanticAI agent with structured outputs
+- Using promptfoo's Python provider to evaluate agents
+- JSON schema validation with `is-json` assertions
+- Multiple assertion types: JavaScript, Python, and LLM-rubric evaluations
+- Evaluating agent tool usage
+
+## Example Structure
+
+- `agent.py` - Simple PydanticAI weather agent with structured output
+- `provider.py` - Promptfoo Python provider that runs the agent
+- `promptfooconfig.yaml` - Evaluation configuration with diverse assertion types
+- `requirements.txt` - Python dependencies

examples/integration-pydantic-ai/agent.py

@@ -0,0 +1,84 @@
+"""
+Simple weather assistant agent using PydanticAI.
+
+This agent demonstrates structured outputs by returning weather information
+in a consistent format using Pydantic models.
+"""
+
+from pydantic import BaseModel
+from pydantic_ai import Agent, RunContext
+
+# Use the OpenAI Responses API with the smallest current GPT-5 reasoning model.
+# The `openai-responses:` prefix selects the Responses API explicitly (the bare
+# `openai:` prefix also resolves there in PydanticAI v2.0+).
+DEFAULT_MODEL = "openai-responses:gpt-5.4-nano"
+
+
+class WeatherResponse(BaseModel):
+    """Structured weather response"""
+
+    location: str
+    temperature: str
+    description: str
+
+
+def get_weather(ctx: RunContext, location: str) -> dict:
+    """Get weather data for a location (mock implementation for demo)"""
+    # Simple mock weather data for demonstration
+    mock_weather = {
+        "london": {"temp": "18°C", "desc": "Cloudy"},
+        "new york": {"temp": "22°C", "desc": "Sunny"},
+        "tokyo": {"temp": "16°C", "desc": "Rainy"},
+    }
+
+    location_lower = location.lower()
+    for city, weather in mock_weather.items():
+        if city in location_lower:
+            return {
+                "location": location,
+                "temperature": weather["temp"],
+                "description": weather["desc"],
+            }
+
+    # Default response for unknown locations
+    return {"location": location, "temperature": "21°C", "description": "Clear"}
+
+
+def get_weather_agent(model: str = DEFAULT_MODEL) -> Agent:
+    """Create a weather agent with structured output"""
+    agent = Agent(
+        model,
+        output_type=WeatherResponse,
+        system_prompt=(
+            "You are a helpful weather assistant. "
+            "Use the get_weather tool to fetch weather data for locations. "
+            "Always return responses in the required structured format."
+        ),
+    )
+
+    agent.tool(get_weather)
+    return agent
+
+
+async def run_weather_agent(query: str, model: str = DEFAULT_MODEL) -> WeatherResponse:
+    """Run the weather agent with a query"""
+    try:
+        agent = get_weather_agent(model)
+        result = await agent.run(query)
+        return result.output
+    except Exception as e:
+        return WeatherResponse(
+            location="Unknown", temperature="N/A", description=f"Error: {str(e)}"
+        )
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def test_agent():
+        queries = ["What's the weather like in London?"]
+        for query in queries:
+            result = await run_weather_agent(query)
+            print(f"{query} -> {result.model_dump_json()}")
+
+    asyncio.run(test_agent())

examples/integration-pydantic-ai/promptfooconfig.yaml

@@ -0,0 +1,45 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: PydanticAI agent evaluation
+
+prompts:
+  - '{{query}}'
+
+providers:
+  - id: file://provider.py
+    label: PydanticAI Weather Agent
+
+defaultTest:
+  assert:
+    - type: is-json
+      value:
+        type: object
+        properties:
+          location:
+            type: string
+          temperature:
+            type: string
+          description:
+            type: string
+        required: ['location', 'temperature', 'description']
+
+tests:
+  - description: 'Basic weather query with JavaScript assertion'
+    vars:
+      query: "What's the weather like in London?"
+    assert:
+      - type: javascript
+        value: output.location.toLowerCase().includes('london') && output.temperature !== 'N/A'
+
+  - description: 'Structured output validation with Python assertion'
+    vars:
+      query: 'Weather in New York'
+    assert:
+      - type: python
+        value: "'new york' in output.get('location', '').lower() and output.get('temperature', '').endswith(('°C', '°F'))"
+
+  - description: 'Weather quality assessment with LLM rubric'
+    vars:
+      query: 'Weather in Tokyo'
+    assert:
+      - type: llm-rubric
+        value: 'The weather response should be realistic and include proper temperature units. The location should be correctly identified as Tokyo.'

examples/integration-pydantic-ai/provider.py

@@ -0,0 +1,44 @@
+"""
+Promptfoo Python provider for PydanticAI agents.
+
+This provider runs PydanticAI agents and returns structured outputs
+for evaluation by promptfoo.
+"""
+
+import asyncio
+import os
+from typing import Any, Dict
+
+from agent import DEFAULT_MODEL, run_weather_agent
+
+
+def call_api(
+    prompt: str, options: Dict[str, Any], context: Dict[str, Any]
+) -> Dict[str, Any]:
+    """Main provider function for PydanticAI weather agent."""
+    try:
+        config = options.get("config", {})
+        model = config.get("model", DEFAULT_MODEL)
+
+        result = asyncio.run(run_weather_agent(prompt, model))
+        output_dict = result.model_dump() if hasattr(result, "model_dump") else result
+
+        return {"output": output_dict}
+
+    except Exception as e:
+        return {
+            "output": {
+                "location": "Unknown",
+                "temperature": "N/A",
+                "description": f"Error: {str(e)}",
+            }
+        }
+
+
+if __name__ == "__main__":
+    print("Testing PydanticAI provider...")
+    if os.getenv("OPENAI_API_KEY"):
+        result = call_api("Weather in London?", {}, {})
+        print(f"Result: {result}")
+    else:
+        print("Set OPENAI_API_KEY to test.")

examples/integration-pydantic-ai/requirements.txt

@@ -0,0 +1,4 @@
+openai==2.38.0
+pydantic==2.13.4
+pydantic-ai==1.102.0
+python-dotenv==1.2.2