chore: import upstream snapshot with attribution

docs/.cursor/rules.mdc

@@ -0,0 +1,6 @@
+---
+description:
+globs:
+alwaysApply: true
+---
+Always reference the python code in the docs e.g. `ModelSettings` should link to [`ModelSettings`][pydantic_ai.settings.ModelSettings].

docs/.hooks/algolia.py

@@ -0,0 +1,185 @@
+# pyright: reportUnknownMemberType=false
+from __future__ import annotations as _annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+from pydantic import TypeAdapter
+from typing_extensions import TypedDict
+
+if TYPE_CHECKING:
+    from mkdocs.config import Config
+    from mkdocs.structure.files import Files
+    from mkdocs.structure.pages import Page
+
+
+class AlgoliaRecord(TypedDict):
+    content: str
+    pageID: str
+    abs_url: str
+    title: str
+    objectID: str
+    rank: int
+
+
+records: list[AlgoliaRecord] = []
+records_ta = TypeAdapter(list[AlgoliaRecord])
+# these values should match docs/javascripts/search-worker.js.
+ALGOLIA_APP_ID = 'KPPUDTIAVX'
+ALGOLIA_INDEX_NAME = 'pydantic-ai-docs'
+
+# Algolia has a limit of 100kb per record in the paid plan,
+# leave some space for the other fields as well.
+MAX_CONTENT_LENGTH = 90_000
+
+
+def on_page_content(html: str, page: Page, config: Config, files: Files) -> str:
+    if not os.getenv('CI'):
+        return html
+
+    from bs4 import BeautifulSoup
+
+    assert page.title is not None, 'Page title must not be None'
+    title = cast(str, page.title)
+
+    soup = BeautifulSoup(html, 'html.parser')
+
+    # If the page does not start with a heading, add the h1 with the title
+    # Some examples don't have a heading. or start with h2
+    first_element = soup.find()
+
+    if not first_element or not first_element.name or first_element.name not in ['h1', 'h2', 'h3']:
+        soup.insert(0, BeautifulSoup(f'<h1 id="{title}">{title}</h1>', 'html.parser'))
+
+    # Clean up presentational and UI elements
+    for element in soup.find_all(['autoref']):
+        element.decompose()
+
+    # this removes the large source code embeds from Github
+    for element in soup.find_all('details'):
+        element.decompose()
+
+    # Cleanup code examples
+    for extra in soup.find_all('div', attrs={'class': 'language-python highlight'}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
+    # Cleanup code examples, part 2
+    for extra in soup.find_all('div', attrs={'class': 'language-python doc-signature highlight'}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
+    # The API reference generates HTML tables with line numbers, this strips the line numbers cell and goes back to a code block
+    for extra in soup.find_all('table', attrs={'class': 'highlighttable'}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
+    headings = soup.find_all(['h1', 'h2', 'h3'])
+
+    # Use the rank to put the sections in the beginning higher in the search results
+    rank = 100
+
+    # Process each section
+    for current_heading in headings:
+        heading_id = current_heading.get('id', '')
+        section_title = current_heading.get_text().replace('¶', '').strip()
+
+        # Get content until next heading
+        content: list[str] = []
+        sibling = current_heading.find_next_sibling()
+        while sibling and sibling.name not in {'h1', 'h2', 'h3'}:
+            content.append(str(sibling))
+            sibling = sibling.find_next_sibling()
+
+        section_html = ''.join(content)
+
+        section_soup = BeautifulSoup(section_html, 'html.parser')
+        section_plain_text = section_soup.get_text(' ', strip=True)
+
+        # Create anchor URL
+        anchor_url: str = f'{page.abs_url}#{heading_id}' if heading_id else page.abs_url or ''
+
+        record_title = title
+
+        if current_heading.name == 'h2':
+            record_title = f'{title} - {section_title}'
+        elif current_heading.name == 'h3':
+            previous_heading = current_heading.find_previous(['h1', 'h2'])
+            record_title = f'{title} - {previous_heading.get_text()} - {section_title}'
+
+        # print(f'Adding record {record_title}, {rank}, {current_heading.name}')
+        # Create record for this section
+        records.append(
+            AlgoliaRecord(
+                content=section_plain_text,
+                pageID=title,
+                abs_url=anchor_url,
+                title=record_title,
+                objectID=anchor_url,
+                rank=rank,
+            )
+        )
+
+        rank -= 5
+
+    return html
+
+
+ALGOLIA_RECORDS_FILE = 'algolia_records.json'
+
+
+def on_post_build(config: Config) -> None:
+    if records:
+        algolia_records_path = Path(config['site_dir']) / ALGOLIA_RECORDS_FILE
+        with algolia_records_path.open('wb') as f:
+            f.write(records_ta.dump_json(records))
+
+
+def algolia_upload() -> None:
+    from algoliasearch.search.client import SearchClientSync
+
+    algolia_write_api_key = os.environ['ALGOLIA_WRITE_API_KEY']
+
+    client = SearchClientSync(ALGOLIA_APP_ID, algolia_write_api_key)
+    filtered_records: list[AlgoliaRecord] = []
+
+    algolia_records_path = Path.cwd() / 'site' / ALGOLIA_RECORDS_FILE
+
+    with algolia_records_path.open('rb') as f:
+        all_records = records_ta.validate_json(f.read())
+
+    for record in all_records:
+        content = record['content']
+        if len(content) > MAX_CONTENT_LENGTH:
+            print(
+                f"Record with title '{record['title']}' has more than {MAX_CONTENT_LENGTH} characters, {len(content)}."
+            )
+            print(content)
+        else:
+            filtered_records.append(record)
+
+    print(f'Uploading {len(filtered_records)} out of {len(all_records)} records to Algolia...')
+
+    client.clear_objects(index_name=ALGOLIA_INDEX_NAME)
+    client.set_settings(
+        index_name=ALGOLIA_INDEX_NAME,
+        index_settings={
+            'searchableAttributes': ['title', 'content'],
+            'attributesToSnippet': ['content:40'],
+            'customRanking': [
+                'desc(rank)',
+            ],
+        },
+    )
+
+    client.batch(
+        index_name=ALGOLIA_INDEX_NAME,
+        batch_write_params={'requests': [{'action': 'addObject', 'body': record} for record in filtered_records]},
+    )
+
+
+if __name__ == '__main__':
+    if sys.argv[-1] == 'upload':
+        algolia_upload()
+    else:
+        print('Run with "upload" argument to upload records to Algolia.')
+        exit(1)

docs/.hooks/main.py

@@ -0,0 +1,250 @@
+from __future__ import annotations as _annotations
+
+import re
+import time
+import urllib.parse
+from pathlib import Path
+
+from jinja2 import Environment
+from mkdocs.config import Config
+from mkdocs.structure.files import Files
+from mkdocs.structure.pages import Page
+from snippets import inject_snippets
+
+DOCS_ROOT = Path(__file__).parent.parent
+
+
+def on_page_markdown(markdown: str, page: Page, config: Config, files: Files) -> str:
+    """Called on each file after it is read and before it is converted to HTML."""
+    relative_path = DOCS_ROOT / page.file.src_uri
+    markdown = inject_snippets(markdown, relative_path.parent)
+    markdown = replace_uv_python_run(markdown)
+    markdown = render_examples(markdown)
+    markdown = render_video(markdown)
+    markdown = create_gateway_toggle(markdown, relative_path)
+    return markdown
+
+
+# path to the main mkdocs material bundle file, found during `on_env`
+bundle_path: Path | None = None
+
+
+def on_env(env: Environment, config: Config, files: Files) -> Environment:
+    global bundle_path
+    for file in files:
+        if re.match('assets/javascripts/bundle.[a-z0-9]+.min.js', file.src_uri):
+            bundle_path = Path(file.dest_dir) / file.src_uri
+
+    env.globals['build_timestamp'] = str(int(time.time()))
+    return env
+
+
+def on_post_build(config: Config) -> None:
+    """Inject extra CSS into mermaid styles to avoid titles being the same color as the background in dark mode."""
+    assert bundle_path is not None
+    if bundle_path.exists():
+        content = bundle_path.read_text(encoding='utf-8')
+        content, _ = re.subn(r'}(\.statediagram)', '}.statediagramTitleText{fill:#888}\1', content, count=1)
+        bundle_path.write_text(content, encoding='utf-8')
+
+
+def replace_uv_python_run(markdown: str) -> str:
+    return re.sub(r'```bash\n(.*?)(python/uv[\- ]run|pip/uv[\- ]add|py-cli)(.+?)\n```', sub_run, markdown)
+
+
+def sub_run(m: re.Match[str]) -> str:
+    prefix = m.group(1)
+    command = m.group(2)
+    if 'pip' in command:
+        pip_base = 'pip install'
+        uv_base = 'uv add'
+    elif command == 'py-cli':
+        pip_base = ''
+        uv_base = 'uv run'
+    else:
+        pip_base = 'python'
+        uv_base = 'uv run'
+    suffix = m.group(3)
+    return f"""\
+=== "pip"
+
+    ```bash
+    {prefix}{pip_base}{suffix}
+    ```
+
+=== "uv"
+
+    ```bash
+    {prefix}{uv_base}{suffix}
+    ```"""
+
+
+EXAMPLES_DIR = Path(__file__).parent.parent.parent / 'examples'
+
+
+def render_examples(markdown: str) -> str:
+    return re.sub(r'^#! *examples/(.+)', sub_example, markdown, flags=re.M)
+
+
+def sub_example(m: re.Match[str]) -> str:
+    example_path = EXAMPLES_DIR / m.group(1)
+    content = example_path.read_text(encoding='utf-8').strip()
+    # remove leading docstring which duplicates what's in the docs page
+    content = re.sub(r'^""".*?"""', '', content, count=1, flags=re.S).strip()
+
+    return content
+
+
+def render_video(markdown: str) -> str:
+    return re.sub(r'\{\{ *video\((["\'])(.+?)\1(?:, (\d+))?(?:, (\d+))?\) *\}\}', sub_cf_video, markdown)
+
+
+def sub_cf_video(m: re.Match[str]) -> str:
+    video_id = m.group(2)
+    time = m.group(3)
+    time = f'{time}s' if time else ''
+    padding_top = m.group(4) or '67'
+
+    domain = 'https://customer-nmegqx24430okhaq.cloudflarestream.com'
+    poster = f'{domain}/{video_id}/thumbnails/thumbnail.jpg?time={time}&height=600'
+    return f"""
+<div style="position: relative; padding-top: {padding_top}%;">
+  <iframe
+    src="{domain}/{video_id}/iframe?poster={urllib.parse.quote_plus(poster)}"
+    loading="lazy"
+    style="border: none; position: absolute; top: 0; left: 0; height: 100%; width: 100%;"
+    allow="accelerometer; gyroscope; autoplay; encrypted-media; picture-in-picture;"
+    allowfullscreen="true"
+  ></iframe>
+</div>
+"""
+
+
+def create_gateway_toggle(markdown: str, relative_path: Path) -> str:
+    """Transform Python code blocks with Agent() calls to show both Pydantic AI and Gateway versions."""
+    # Pattern matches Python code blocks with or without attributes, and optional annotation definitions after.
+    # Captures leading indentation (group 1) to support code blocks nested inside admonitions or other contexts.
+    # The closing fence must match the same indentation via backreference \1.
+    # Annotation definitions are numbered list items like "1. Some text" that follow the code block.
+    return re.sub(
+        r'^( *)```py(?:thon)?(?: *\{?([^}\n]*)\}?)?\n(.*?)\n\1```(\n\n(?:[ \t]*\d+\.[^\n]+\n)+\n)?',
+        lambda m: transform_gateway_code_block(m, relative_path),
+        markdown,
+        flags=re.MULTILINE | re.DOTALL,
+    )
+
+
+# Mapping of user-facing provider prefixes in docs to the form shown in the Gateway tab.
+# 1-to-1 identity: the toggle shows `Agent('openai:')` ↔ `Agent('gateway/openai:')`.
+# Wire-value translation (e.g. `openai` -> `openai-responses` on the Gateway URL) is the
+# runtime's concern and lives in `providers/gateway.py::_GATEWAY_ROUTE_REMAP`.
+GATEWAY_MODEL_MAP = {
+    'anthropic': 'anthropic',
+    'openai': 'openai',
+    'openai-responses': 'openai-responses',
+    'openai-chat': 'openai-chat',
+    'bedrock': 'bedrock',
+    'google': 'google',
+    'google-cloud': 'google-cloud',
+    'groq': 'groq',
+}
+# Models that should get gateway transformation
+GATEWAY_MODELS = tuple(GATEWAY_MODEL_MAP.keys())
+
+
+def transform_gateway_code_block(m: re.Match[str], relative_path: Path) -> str:
+    """Transform a single code block to show both versions if it contains Agent() calls."""
+    indent = m.group(1)  # Leading whitespace (e.g. 4 spaces when inside an admonition)
+    attrs = m.group(2) or ''
+    code = m.group(3)
+    annotations = m.group(4) or ''  # Capture annotation definitions if present
+
+    tab_indent = indent + '    '  # Tab content needs 4 more spaces than the surrounding context
+
+    # Simple check: does the code contain both "Agent(" and a quoted string?
+    if 'Agent(' not in code:
+        attrs_str = f' {{{attrs}}}' if attrs else ''
+        return f'{indent}```python{attrs_str}\n{code}\n{indent}```{annotations}'
+
+    # Check if code contains Agent() with a model that should be transformed
+    # Look for Agent(...'model:...' or Agent(..."model:..."
+    agent_pattern = r'Agent\((?:(?!["\']).)*([\"\'])([^"\']+)\1'
+    agent_match = re.search(agent_pattern, code, flags=re.DOTALL)
+
+    if not agent_match:
+        # No Agent() with string literal found
+        attrs_str = f' {{{attrs}}}' if attrs else ''
+        return f'{indent}```python{attrs_str}\n{code}\n{indent}```{annotations}'
+
+    model_string = agent_match.group(2)
+    # Check if model starts with one of the gateway-supported models
+    should_transform = any(model_string.startswith(f'{model}:') for model in GATEWAY_MODELS)
+
+    if not should_transform:
+        # Model doesn't match gateway models, return original
+        attrs_str = f' {{{attrs}}}' if attrs else ''
+        return f'{indent}```python{attrs_str}\n{code}\n{indent}```{annotations}'
+
+    # Transform the code for gateway version
+    def replace_agent_model(match: re.Match[str]) -> str:
+        """Replace model string with gateway/ prefix if it's a supported provider."""
+        full_match = match.group(0)
+        quote = match.group(1)
+        model = match.group(2)
+
+        for provider, gateway_provider in GATEWAY_MODEL_MAP.items():
+            if model.startswith(f'{provider}:'):
+                new_model = model.replace(f'{provider}:', f'gateway/{gateway_provider}:', 1)
+                return full_match.replace(f'{quote}{model}{quote}', f'{quote}{new_model}{quote}', 1)
+
+        return full_match
+
+    # This pattern finds: "Agent(" followed by anything (lazy), then the first quoted string
+    gateway_code = re.sub(
+        agent_pattern,
+        replace_agent_model,
+        code,
+        flags=re.DOTALL,
+    )
+
+    # Build attributes string
+    docs_path = DOCS_ROOT / 'gateway'
+
+    relative_path_to_gateway = docs_path.relative_to(relative_path, walk_up=True)
+    link = f"<a href='{relative_path_to_gateway}' style='float: right;'>Learn about Gateway</a>"
+    attrs_str = f' {{{attrs}}}' if attrs else ''
+
+    if 'title="' in attrs:
+        gateway_attrs = attrs.replace('title="', f'title="{link} ', 1)
+    else:
+        gateway_attrs = attrs + f' title="{link}"'
+    gateway_attrs_str = f' {{{gateway_attrs}}}'
+
+    # Indent code lines for the tab content (add 4 spaces on top of any existing indent).
+    # Code lines already carry the outer `indent` prefix from the source, so adding 4 more
+    # spaces brings them to the correct `tab_indent` level inside the generated tab block.
+    code_lines = code.split('\n')
+    indented_code = '\n'.join('    ' + line for line in code_lines)
+
+    gateway_code_lines = gateway_code.split('\n')
+    indented_gateway_code = '\n'.join('    ' + line for line in gateway_code_lines)
+
+    # Indent annotation definitions if present (need to be inside tabs for Material to work).
+    # Annotation lines already carry the outer `indent` prefix, so adding 4 more spaces is correct.
+    indented_annotations = ''
+    if annotations:
+        annotation_lines = annotations.strip().split('\n')
+        indented_annotations = '\n\n' + '\n'.join('    ' + line for line in annotation_lines) + '\n\n'
+
+    return f"""\
+{indent}=== "With Pydantic AI Gateway"
+
+{tab_indent}```python{gateway_attrs_str}
+{indented_gateway_code}
+{tab_indent}```{indented_annotations}
+
+{indent}=== "Directly to Provider API"
+
+{tab_indent}```python{attrs_str}
+{indented_code}
+{tab_indent}```{indented_annotations}"""

docs/.hooks/snippets.py

@@ -0,0 +1,299 @@
+from __future__ import annotations as _annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).parent.parent.parent
+PYDANTIC_AI_EXAMPLES_ROOT = REPO_ROOT / 'examples' / 'pydantic_ai_examples'
+
+
+@dataclass
+class SnippetDirective:
+    path: str
+    title: str | None = None
+    fragment: str | None = None
+    highlight: str | None = None
+    extra_attrs: dict[str, str] | None = None
+
+
+@dataclass
+class LineRange:
+    start_line: int  # first line in file is 0
+    end_line: int  # unlike start_line, this line is interpreted as excluded from the range; this should always be larger than the start_line
+
+    def intersection(self, ranges: list[LineRange]) -> list[LineRange]:
+        new_ranges: list[LineRange] = []
+        for r in ranges:
+            new_start_line = max(r.start_line, self.start_line)
+            new_end_line = min(r.end_line, self.end_line)
+            if new_start_line < new_end_line:
+                new_ranges.append(r)
+        return new_ranges
+
+    @staticmethod
+    def merge(ranges: list[LineRange]) -> list[LineRange]:
+        if not ranges:
+            return []
+
+        # Sort ranges by start_line
+        sorted_ranges = sorted(ranges, key=lambda r: r.start_line)
+        merged: list[LineRange] = []
+
+        for current in sorted_ranges:
+            if not merged or merged[-1].end_line < current.start_line:
+                # No overlap with previous range, add as new range
+                merged.append(current)
+            else:
+                # Overlap or adjacent, merge with previous range
+                merged[-1] = LineRange(merged[-1].start_line, max(merged[-1].end_line, current.end_line))
+
+        return merged
+
+
+@dataclass
+class RenderedSnippet:
+    content: str
+    highlights: list[LineRange]
+    original_range: LineRange
+
+
+@dataclass
+class ParsedFile:
+    lines: list[str]
+    sections: dict[str, list[LineRange]]
+    lines_mapping: dict[int, int]
+
+    def render(self, fragment_sections: list[str], highlight_sections: list[str]) -> RenderedSnippet:
+        fragment_ranges: list[LineRange] = []
+        if fragment_sections:
+            for k in fragment_sections:
+                if k not in self.sections:
+                    raise ValueError(f'Unrecognized fragment section: {k!r} (expected {list(self.sections)})')
+                fragment_ranges.extend(self.sections[k])
+            fragment_ranges = LineRange.merge(fragment_ranges)
+        else:
+            fragment_ranges = [LineRange(0, len(self.lines))]
+
+        highlight_ranges: list[LineRange] = []
+        for k in highlight_sections:
+            if k not in self.sections:
+                raise ValueError(f'Unrecognized highlight section: {k!r} (expected {list(self.sections)})')
+            highlight_ranges.extend(self.sections[k])
+        highlight_ranges = LineRange.merge(highlight_ranges)
+
+        rendered_highlight_ranges = list[LineRange]()
+        rendered_lines: list[str] = []
+        last_end_line = 1
+        current_line = 0
+        for fragment_range in fragment_ranges:
+            if fragment_range.start_line > last_end_line:
+                if current_line == 0:
+                    rendered_lines.append('...\n')
+                else:
+                    rendered_lines.append('\n...\n')
+
+                current_line += 1
+            fragment_highlight_ranges = fragment_range.intersection(highlight_ranges)
+            for fragment_highlight_range in fragment_highlight_ranges:
+                rendered_highlight_ranges.append(
+                    LineRange(
+                        fragment_highlight_range.start_line - fragment_range.start_line + current_line,
+                        fragment_highlight_range.end_line - fragment_range.start_line + current_line,
+                    )
+                )
+
+            for i in range(fragment_range.start_line, fragment_range.end_line):
+                rendered_lines.append(self.lines[i])
+                current_line += 1
+            last_end_line = fragment_range.end_line
+
+        if last_end_line < len(self.lines):
+            rendered_lines.append('\n...')
+
+        original_range = LineRange(
+            self.lines_mapping[fragment_ranges[0].start_line],
+            self.lines_mapping[fragment_ranges[-1].end_line - 1] + 1,
+        )
+        return RenderedSnippet('\n'.join(rendered_lines), LineRange.merge(rendered_highlight_ranges), original_range)
+
+
+def parse_snippet_directive(line: str) -> SnippetDirective | None:
+    """Parse a line like: ```snippet {path="..." title="..." fragment="..." highlight="..."}```"""
+    pattern = r'```snippet\s+\{([^}]+)\}'
+    match = re.match(pattern, line.strip())
+    if not match:
+        return None
+
+    attrs_str = match.group(1)
+    attrs: dict[str, str] = {}
+
+    # Parse key="value" pairs
+    for attr_match in re.finditer(r'(\w+)="([^"]*)"', attrs_str):
+        key, value = attr_match.groups()
+        attrs[key] = value
+
+    if 'path' not in attrs:
+        raise ValueError('Missing required key "path" in snippet directive')
+
+    extra_attrs = {k: v for k, v in attrs.items() if k not in ['path', 'title', 'fragment', 'highlight']}
+
+    return SnippetDirective(
+        path=attrs['path'],
+        title=attrs.get('title'),
+        fragment=attrs.get('fragment'),
+        highlight=attrs.get('highlight'),
+        extra_attrs=extra_attrs if extra_attrs else None,
+    )
+
+
+def parse_file_sections(file_path: Path) -> ParsedFile:
+    """Parse a file and extract sections marked with ### [section] or /// [section]"""
+    input_lines = file_path.read_text(encoding='utf-8').splitlines()
+    output_lines: list[str] = []
+    lines_mapping: dict[int, int] = {}
+
+    sections: dict[str, list[LineRange]] = {}
+    section_starts: dict[str, int] = {}
+
+    output_line_no = 0
+    for line_no, line in enumerate(input_lines, 1):
+        match: re.Match[str] | None = None
+        for match in re.finditer(r'\s*(?:###|///)\s*\[([^]]+)]\s*$', line):
+            break
+        else:
+            output_lines.append(line)
+            output_line_no += 1
+            lines_mapping[output_line_no - 1] = line_no - 1
+            continue
+
+        pre_matches_line = line[: match.start()]
+        sections_to_start: set[str] = set()
+        sections_to_end: set[str] = set()
+        for item in match.group(1).split(','):
+            if item in sections_to_end or item in sections_to_start:
+                raise ValueError(f'Duplicate section reference: {item!r} at {file_path}:{line_no}')
+            if item.startswith('/'):
+                sections_to_end.add(item[1:])
+            else:
+                sections_to_start.add(item)
+
+        for section_name in sections_to_start:
+            if section_name in section_starts:
+                raise ValueError(f'Cannot nest section with the same name {section_name!r} at {file_path}:{line_no}')
+            section_starts[section_name] = output_line_no
+
+        for section_name in sections_to_end:
+            start_line = section_starts.pop(section_name, None)
+            if start_line is None:
+                raise ValueError(f'Cannot end unstarted section {section_name!r} at {file_path}:{line_no}')
+            if section_name not in sections:
+                sections[section_name] = []
+            end_line = output_line_no + 1 if pre_matches_line else output_line_no
+            sections[section_name].append(LineRange(start_line, end_line))
+
+        if pre_matches_line:
+            output_lines.append(pre_matches_line)
+            output_line_no += 1
+            lines_mapping[output_line_no - 1] = line_no - 1
+
+    if section_starts:
+        raise ValueError(f'Some sections were not finished in {file_path}: {list(section_starts)}')
+
+    return ParsedFile(lines=output_lines, sections=sections, lines_mapping=lines_mapping)
+
+
+def format_highlight_lines(highlight_ranges: list[LineRange]) -> str:
+    """Convert highlight ranges to mkdocs hl_lines format"""
+    if not highlight_ranges:
+        return ''
+
+    parts: list[str] = []
+    for range in highlight_ranges:
+        start = range.start_line + 1  # convert to 1-based indexing
+        end = range.end_line  # SectionRanges exclude the end, so just don't add 1 here
+        if start == end:
+            parts.append(str(start))
+        else:
+            parts.append(f'{start}-{end}')
+
+    return ' '.join(parts)
+
+
+def inject_snippets(markdown: str, relative_path_root: Path) -> str:  # noqa: C901
+    def replace_snippet(match: re.Match[str]) -> str:
+        line = match.group(0)
+        directive = parse_snippet_directive(line)
+        if not directive:
+            return line
+
+        if directive.path.startswith('/'):
+            # If directive path is absolute, treat it as relative to the repo root:
+            file_path = (REPO_ROOT / directive.path[1:]).resolve()
+        else:
+            # Else, resolve as a relative path
+            file_path = (relative_path_root / directive.path).resolve()
+
+        if not file_path.exists():
+            raise FileNotFoundError(f'File {file_path} not found')
+
+        # Parse the file sections
+        parsed_file = parse_file_sections(file_path)
+
+        # Determine fragments to extract
+        fragment_names = directive.fragment.split() if directive.fragment else []
+        highlight_names = directive.highlight.split() if directive.highlight else []
+
+        # Extract content
+        rendered = parsed_file.render(fragment_names, highlight_names)
+
+        # Get file extension for syntax highlighting
+        file_extension = file_path.suffix.lstrip('.')
+
+        # Determine title
+        if directive.title:
+            title = directive.title
+        else:
+            if file_path.is_relative_to(PYDANTIC_AI_EXAMPLES_ROOT):
+                title_path = str(file_path.relative_to(PYDANTIC_AI_EXAMPLES_ROOT))
+            else:
+                title_path = file_path.name
+            title = title_path
+            range_spec: str | None = None
+            if directive.fragment:
+                range_spec = f'L{rendered.original_range.start_line + 1}-L{rendered.original_range.end_line}'
+                title = f'{title_path} ({range_spec})'
+            if file_path.is_relative_to(REPO_ROOT):
+                relative_path = file_path.relative_to(REPO_ROOT)
+                url = f'https://github.com/pydantic/pydantic-ai/blob/main/{relative_path}'
+                if range_spec is not None:
+                    url += f'#{range_spec}'
+                title = f"<a href='{url}' target='_blank' rel='noopener noreferrer'>{title}</a>"
+        # Build attributes for the code block
+        attrs: list[str] = []
+        if title:
+            attrs.append(f'title="{title}"')
+
+        # Add highlight lines
+        if rendered.highlights:
+            hl_lines = format_highlight_lines(rendered.highlights)
+            if hl_lines:
+                attrs.append(f'hl_lines="{hl_lines}"')
+
+        # Add extra attributes
+        if directive.extra_attrs:
+            for key, value in directive.extra_attrs.items():
+                attrs.append(f'{key}="{value}"')
+
+        # Build the replacement
+        attrs_str = ' '.join(attrs)
+        if attrs_str:
+            attrs_str = ' {' + attrs_str + '}'
+
+        result = f'```{file_extension}{attrs_str}\n{rendered.content}\n```'
+
+        return result
+
+    # Find and replace all snippet directives
+    pattern = r'^```snippet\s+\{[^}]+\}```$'
+    return re.sub(pattern, replace_snippet, markdown, flags=re.MULTILINE)

docs/.hooks/test_snippets.py

@@ -0,0 +1,650 @@
+from __future__ import annotations as _annotations
+
+import os
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+from snippets import (
+    REPO_ROOT,
+    LineRange,
+    ParsedFile,
+    RenderedSnippet,
+    SnippetDirective,
+    format_highlight_lines,
+    inject_snippets,
+    parse_file_sections,
+    parse_snippet_directive,
+)
+
+from tests._inline_snapshot import snapshot
+
+
+@contextmanager
+def temp_text_file(content: str):
+    """Context manager for temporary text file with common params."""
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.py', encoding='utf-8', delete=False) as f:
+        f.write(content)
+        temp_name = f.name
+
+    try:
+        yield Path(temp_name)
+    finally:
+        os.unlink(temp_name)
+
+
+def test_parse_snippet_directive_basic():
+    """Test basic parsing of snippet directives."""
+    line = '```snippet {path="test.py"}```'
+    result = parse_snippet_directive(line)
+    assert result == snapshot(
+        SnippetDirective(path='test.py', title=None, fragment=None, highlight=None, extra_attrs=None)
+    )
+
+
+def test_parse_snippet_directive_all_attrs():
+    """Test parsing with all standard attributes."""
+    line = '```snippet {path="src/main.py" title="Main Module" fragment="init setup" highlight="error-handling"}'
+    result = parse_snippet_directive(line)
+    assert result == snapshot(
+        SnippetDirective(
+            path='src/main.py', title='Main Module', fragment='init setup', highlight='error-handling', extra_attrs=None
+        )
+    )
+
+
+def test_parse_snippet_directive_extra_attrs():
+    """Test parsing with extra attributes."""
+    line = '```snippet {path="test.py" custom="value" another="attr"}'
+    result = parse_snippet_directive(line)
+    assert result == snapshot(
+        SnippetDirective(
+            path='test.py',
+            title=None,
+            fragment=None,
+            highlight=None,
+            extra_attrs={'another': 'attr', 'custom': 'value'},
+        )
+    )
+
+
+def test_parse_snippet_directive_missing_path():
+    """Test that missing path raises ValueError."""
+    line = '```snippet {title="Test"}'
+    with pytest.raises(ValueError, match='Missing required key "path" in snippet directive'):
+        parse_snippet_directive(line)
+
+
+def test_parse_snippet_directive_invalid_format():
+    """Test that invalid format returns None."""
+    assert parse_snippet_directive('```python') is None
+    assert parse_snippet_directive("snippet {path='test.py'}") is None
+    assert parse_snippet_directive('```snippet') is None
+    assert parse_snippet_directive('```snippet```') is None
+
+
+def test_parse_snippet_directive_whitespace():
+    """Test parsing with various whitespace."""
+    line = '   ```snippet   {   path="test.py"   }   '
+    result = parse_snippet_directive(line)
+    assert result == snapshot(
+        SnippetDirective(path='test.py', title=None, fragment=None, highlight=None, extra_attrs=None)
+    )
+
+
+def test_parse_file_sections_basic():
+    """Test basic section parsing."""
+    content = """line 1
+### [section1]
+content 1
+content 2
+### [/section1]
+line 6"""
+
+    with temp_text_file(content) as temp_path:
+        result = parse_file_sections(temp_path)
+
+    assert result == snapshot(
+        ParsedFile(
+            lines=['line 1', 'content 1', 'content 2', 'line 6'],
+            sections={'section1': [LineRange(start_line=1, end_line=3)]},
+            lines_mapping={0: 0, 1: 2, 2: 3, 3: 5},
+        )
+    )
+
+
+def test_parse_file_sections_multiple_ranges():
+    """Test section with multiple disjoint ranges."""
+    content = """line 1
+### [section1]
+content 1
+### [/section1]
+middle line
+### [section1]
+content 2
+### [/section1]
+end line"""
+
+    with temp_text_file(content) as temp_path:
+        result = parse_file_sections(temp_path)
+
+    assert result == snapshot(
+        ParsedFile(
+            lines=[
+                'line 1',
+                'content 1',
+                'middle line',
+                'content 2',
+                'end line',
+            ],
+            sections={'section1': [LineRange(start_line=1, end_line=2), LineRange(start_line=3, end_line=4)]},
+            lines_mapping={0: 0, 1: 2, 2: 4, 3: 6, 4: 8},
+        )
+    )
+
+
+def test_parse_file_sections_comment_style():
+    """Test parsing with /// comment style."""
+    content = """line 1
+/// [section1]
+content 1
+/// [/section1]
+line 5"""
+
+    with temp_text_file(content) as temp_path:
+        result = parse_file_sections(temp_path)
+
+    assert result == snapshot(
+        ParsedFile(
+            lines=['line 1', 'content 1', 'line 5'],
+            sections={'section1': [LineRange(start_line=1, end_line=2)]},
+            lines_mapping={0: 0, 1: 2, 2: 4},
+        )
+    )
+
+
+def test_parse_file_sections_nested():
+    """Test nested sections with different names."""
+    content = """line 1
+### [outer]
+outer content
+### [inner]
+inner content
+### [/inner]
+more outer
+### [/outer]
+end"""
+
+    with temp_text_file(content) as temp_path:
+        result = parse_file_sections(temp_path)
+
+    assert result == snapshot(
+        ParsedFile(
+            lines=[
+                'line 1',
+                'outer content',
+                'inner content',
+                'more outer',
+                'end',
+            ],
+            sections={
+                'inner': [LineRange(start_line=2, end_line=3)],
+                'outer': [LineRange(start_line=1, end_line=4)],
+            },
+            lines_mapping={0: 0, 1: 2, 2: 4, 3: 6, 4: 8},
+        )
+    )
+
+
+def test_extract_fragment_content_entire_file():
+    """Test extracting entire file when no fragments specified."""
+    content = """line 1
+### [section1]
+content 1
+### [/section1]
+line 5"""
+
+    with temp_text_file(content) as temp_path:
+        parsed = parse_file_sections(temp_path)
+
+    assert parsed.render([], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+line 5\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=0, end_line=5),
+        )
+    )
+
+    assert parsed.render(['section1'], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+
+...\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=2, end_line=3),
+        )
+    )
+
+    assert parsed.render([], ['section1']) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+line 5\
+""",
+            highlights=[LineRange(start_line=1, end_line=2)],
+            original_range=LineRange(start_line=0, end_line=5),
+        )
+    )
+
+
+def test_extract_fragment_content_specific_section():
+    """Test extracting specific section."""
+    content = """line 1
+### [section1]
+content 1
+content 2
+### [/section1]
+line 6"""
+
+    with temp_text_file(content) as temp_path:
+        parsed = parse_file_sections(temp_path)
+
+    assert parsed.render([], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+content 2
+line 6\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=0, end_line=6),
+        )
+    )
+
+    assert parsed.render(['section1'], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+content 2
+
+...\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=2, end_line=4),
+        )
+    )
+
+    assert parsed.render([], ['section1']) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+content 2
+line 6\
+""",
+            highlights=[LineRange(start_line=1, end_line=3)],
+            original_range=LineRange(start_line=0, end_line=6),
+        )
+    )
+
+
+def test_extract_fragment_content_multiple_sections():
+    """Test extracting multiple disjoint sections."""
+    content = """line 1
+### [section1]
+content 1
+### [/section1]
+middle
+### [section2]
+content 2
+### [/section2]
+end"""
+
+    with temp_text_file(content) as temp_path:
+        parsed = parse_file_sections(temp_path)
+
+    assert parsed.render([], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+middle
+content 2
+end\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=0, end_line=9),
+        )
+    )
+
+    assert parsed.render(['section1', 'section2'], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+
+...
+
+content 2
+
+...\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=2, end_line=7),
+        )
+    )
+
+    assert parsed.render(['section1', 'section2'], ['section1']) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+
+...
+
+content 2
+
+...\
+""",
+            highlights=[LineRange(start_line=0, end_line=1)],
+            original_range=LineRange(start_line=2, end_line=7),
+        )
+    )
+
+    assert parsed.render(['section1', 'section2'], ['section1', 'section2']) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+
+...
+
+content 2
+
+...\
+""",
+            highlights=[LineRange(start_line=0, end_line=1), LineRange(start_line=2, end_line=3)],
+            original_range=LineRange(start_line=2, end_line=7),
+        )
+    )
+
+    assert parsed.render(['section1'], ['section2']) == snapshot(
+        RenderedSnippet(
+            content="""\
+content 1
+
+...\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=2, end_line=3),
+        )
+    )
+
+    assert parsed.render([], ['section1', 'section2']) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+content 1
+middle
+content 2
+end\
+""",
+            highlights=[LineRange(start_line=1, end_line=2), LineRange(start_line=3, end_line=4)],
+            original_range=LineRange(start_line=0, end_line=9),
+        )
+    )
+
+
+def test_complicated_example():
+    """Test extracting multiple overlapping sections."""
+    content = """line 1
+### [fragment1]
+line 2
+### [fragment2]
+line 3
+### [highlight1,highlight2]
+line 4
+### [/fragment1,/highlight1]
+line 5
+### [/fragment2]
+line 6
+### [/highlight2]
+"""
+
+    with temp_text_file(content) as temp_path:
+        parsed = parse_file_sections(temp_path)
+
+    assert parsed.render([], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 1
+line 2
+line 3
+line 4
+line 5
+line 6\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=0, end_line=11),
+        )
+    )
+
+    assert parsed.render(['fragment1'], ['highlight1']) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 2
+line 3
+line 4
+
+...\
+""",
+            highlights=[LineRange(start_line=2, end_line=3)],
+            original_range=LineRange(start_line=2, end_line=7),
+        )
+    )
+
+    assert parsed.render(['fragment1'], ['highlight2']) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 2
+line 3
+line 4
+
+...\
+""",
+            highlights=[LineRange(start_line=2, end_line=5)],
+            original_range=LineRange(start_line=2, end_line=7),
+        )
+    )
+
+    assert parsed.render(['fragment2'], ['highlight2']) == snapshot(
+        RenderedSnippet(
+            content="""\
+...
+
+line 3
+line 4
+line 5
+
+...\
+""",
+            highlights=[LineRange(start_line=2, end_line=5)],
+            original_range=LineRange(start_line=4, end_line=9),
+        )
+    )
+
+    assert parsed.render(['fragment1', 'fragment2'], []) == snapshot(
+        RenderedSnippet(
+            content="""\
+line 2
+line 3
+line 4
+line 5
+
+...\
+""",
+            highlights=[],
+            original_range=LineRange(start_line=2, end_line=9),
+        )
+    )
+
+
+def test_format_highlight_lines_empty():
+    """Test formatting empty highlight ranges."""
+    assert format_highlight_lines([]) == ''
+
+
+def test_format_highlight_lines_single():
+    """Test formatting single line highlight."""
+    assert format_highlight_lines([LineRange(0, 1)]) == '1'
+    assert format_highlight_lines([LineRange(5, 6)]) == '6'
+
+
+def test_format_highlight_lines_range():
+    """Test formatting line range highlight."""
+    assert format_highlight_lines([LineRange(0, 3)]) == '1-3'
+    assert format_highlight_lines([LineRange(5, 9)]) == '6-9'
+
+
+def test_format_highlight_lines_multiple():
+    """Test formatting multiple highlights."""
+    assert format_highlight_lines([LineRange(0, 1), LineRange(2, 5), LineRange(6, 7)]) == '1 3-5 7'
+
+
+def test_inject_snippets_basic():
+    """Test basic snippet injection."""
+    content = """def hello():
+    return "world" """
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+
+        # Mock the docs directory resolution by copying file
+        target_file = docs_dir / 'test.py'
+        target_file.write_text(content, encoding='utf-8')
+
+        markdown = '```snippet {path="test.py"}'
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot('```snippet {path="test.py"}')
+
+
+def test_inject_snippets_with_title():
+    """Test snippet injection with custom title."""
+    content = "print('hello')"
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+        target_file = docs_dir / 'test.py'
+        target_file.write_text(content, encoding='utf-8')
+
+        markdown = '```snippet {path="test.py" title="Custom Title"}'
+
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot('```snippet {path="test.py" title="Custom Title"}')
+
+
+def test_inject_snippets_with_fragments():
+    """Test snippet injection with fragments."""
+    content = """line 1
+### [important]
+key_function()
+### [/important]
+line 5"""
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+        target_file = docs_dir / 'test.py'
+        target_file.write_text(content, encoding='utf-8')
+
+        markdown = '```snippet {path="test.py" fragment="important"}'
+
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot('```snippet {path="test.py" fragment="important"}')
+
+
+def test_inject_snippets_with_highlights():
+    """Test snippet injection with highlights."""
+    content = """def normal():
+    pass
+
+### [important]
+def important():
+    return True
+### [/important]
+
+def other():
+    pass"""
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+        target_file = docs_dir / 'test.py'
+        target_file.write_text(content, encoding='utf-8')
+
+        markdown = '```snippet {path="test.py" highlight="important"}'
+
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot('```snippet {path="test.py" highlight="important"}')
+
+
+def test_inject_snippets_nonexistent_file():
+    """Test that nonexistent files raise an error.."""
+    markdown = '```snippet {path="nonexistent.py"}```'
+    with pytest.raises(FileNotFoundError):
+        inject_snippets(markdown, REPO_ROOT)
+
+
+def test_inject_snippets_multiple():
+    """Test injecting multiple snippets in one markdown."""
+    content1 = "print('file1')"
+    content2 = "print('file2')"
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+        file1 = docs_dir / 'test1.py'
+        file2 = docs_dir / 'test2.py'
+        file1.write_text(content1, encoding='utf-8')
+        file2.write_text(content2, encoding='utf-8')
+
+        markdown = """Some text
+```snippet {path="test1.py"}
+More text
+```snippet {path="test2.py"}
+Final text"""
+
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot(
+        """\
+Some text
+```snippet {path="test1.py"}
+More text
+```snippet {path="test2.py"}
+Final text\
+"""
+    )
+
+
+def test_inject_snippets_extra_attrs():
+    """Test snippet injection with extra attributes."""
+    content = "print('test')"
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        docs_dir = Path(temp_dir)
+        target_file = docs_dir / 'test.py'
+        target_file.write_text(content, encoding='utf-8')
+
+        markdown = '```snippet {path="test.py" custom="value" another="attr"}'
+
+        result = inject_snippets(markdown, docs_dir)
+
+    assert result == snapshot('```snippet {path="test.py" custom="value" another="attr"}')

docs/.overrides/.icons/logfire/logo.svg

@@ -0,0 +1,6 @@
+<svg viewBox="0 0 120 120" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="120" height="120" fill="#E520E9"/>
+<g transform="translate(15, 15) scale(0.647)">
+<path fill="#092224" d="M137.542 90.563 73.808 2.241c-2.006-2.757-6.632-2.757-8.617 0L1.456 90.563a5.318 5.318 0 0 0-.998 3.101 5.331 5.331 0 0 0 3.642 5.05l63.735 20.851h.01a5.31 5.31 0 0 0 3.293 0h.01l63.735-20.85a5.265 5.265 0 0 0 3.393-3.406 5.244 5.244 0 0 0-.749-4.746h.015Zm-68.04-76.151 25.545 35.403-23.889-7.813c-.184-.06-.38-.05-.564-.094a3.488 3.488 0 0 0-.549-.09c-.184-.025-.359-.095-.543-.095-.185 0-.355.07-.54.095-.184.02-.368.05-.548.09-.19.035-.384.035-.554.094L44.115 49.77l-.15.05L69.513 14.41h-.01ZM33.408 64.438l27.811-9.104 2.969-.967v52.838L14.324 90.887l19.084-26.449Zm41.412 42.757V54.367l30.78 10.071 19.085 26.434-49.87 16.323h.005Z"/>
+</g>
+</svg>

docs/.overrides/partials/search.html

@@ -0,0 +1,32 @@
+<div class="md-search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" id="searchbox" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" required="">
+      <label class="md-search__icon md-icon" for="__search">
+
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"></path></svg>
+
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"></path></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+
+        <button id="searchbox-clear" type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"></path></svg>
+        </button>
+      </nav>
+
+        <div class="md-search__suggest"></div>
+
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0">
+        <div class="md-search-result">
+          <div class="md-search-result__meta" id="type-to-start-searching">Type to start searching</div>
+          <ol class="md-search-result__list" id="hits" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

docs/.partials/index-header.html

@@ -0,0 +1,38 @@
+<div class="text-center">
+  <img class="index-header off-glb" src="./img/pydantic-ai-dark.svg#only-dark" alt="Pydantic AI">
+</div>
+<div class="text-center">
+  <img class="index-header off-glb" src="./img/pydantic-ai-light.svg#only-light" alt="Pydantic AI">
+</div>
+<p class="text-center">
+  <em>GenAI Agent Framework, the Pydantic way</em>
+</p>
+<p class="text-center">
+  <a href="https://github.com/pydantic/pydantic-ai/actions/workflows/ci.yml?query=branch%3Amain">
+    <img
+      src="https://github.com/pydantic/pydantic-ai/actions/workflows/ci.yml/badge.svg?event=push"
+      alt="CI">
+  </a>
+  <a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/pydantic/pydantic-ai">
+    <img
+      src="https://coverage-badge.samuelcolvin.workers.dev/pydantic/pydantic-ai.svg"
+      alt="Coverage">
+  </a>
+  <a href="https://pypi.python.org/pypi/pydantic-ai">
+    <img src="https://img.shields.io/pypi/v/pydantic-ai.svg" alt="PyPI">
+  </a>
+  <a href="https://github.com/pydantic/pydantic-ai">
+    <img src="https://img.shields.io/pypi/pyversions/pydantic-ai.svg" alt="versions">
+  </a>
+  <a href="https://github.com/pydantic/pydantic-ai/blob/main/LICENSE">
+    <img src="https://img.shields.io/github/license/pydantic/pydantic-ai.svg" alt="license">
+  </a>
+  <a href="https://logfire.pydantic.dev/docs/join-slack/">
+    <img src="https://img.shields.io/badge/Slack-Join%20Slack-4A154B?logo=slack" alt="Join Slack">
+  </a>
+</p>
+
+<p class="text-emphasis">
+  Pydantic AI is a Python agent framework designed to help you
+  quickly, confidently, and painlessly build production grade applications and workflows with Generative AI.
+</p>

docs/AGENTS.md

@@ -0,0 +1,58 @@
+<!-- braindump: rules extracted from PR review patterns -->
+
+# docs/ Guidelines
+
+## Documentation
+
+<!-- rule:232 -->
+- Link all concepts, features, and API elements to their docs/reference pages using anchor fragments (`#section-name`) for specific sections — Improves discoverability and reduces user friction by providing direct navigation to relevant documentation context
+<!-- rule:66 -->
+- Use reference-style links for API elements: `[ElementName][module.path.ElementName]` — enables hover docs and navigation in mkdocs — Provides interactive documentation features like tooltips and jump-to-definition that plain backticks cannot support
+<!-- rule:714 -->
+- Omit deprecated features from user-facing docs — document only current approaches — Prevents users from learning outdated patterns and reduces confusion about the recommended way forward
+<!-- rule:82 -->
+- Write project name as `Pydantic AI` (two words) in docs — not `Pydantic-AI`, `PydanticAI`, or `pAI` — Maintains consistent brand identity and prevents confusion across documentation
+<!-- rule:359 -->
+- Structure code examples as: context/intro → code block → caveats/details (never code before context) — Ensures readers understand purpose and usage before seeing code, making docs more learnable and preventing confusion
+<!-- rule:93 -->
+- Hide implementation details from user docs unless they affect user decisions — focus on what users can control, not how it works internally — Keeps documentation clean and maintainable by separating user-facing APIs from implementation that may change
+<!-- rule:52 -->
+- Structure docs with progressive disclosure: concept → capabilities → examples (standalone first) → config → edge cases — Helps readers build mental models incrementally, reducing cognitive load and making features easier to adopt
+<!-- rule:152 -->
+- In docs, show the **recommended approach first**, then introduce alternatives with explicit relational language ("In addition to...", "As an alternative to...") using specific feature names — Prevents users from adopting legacy or suboptimal patterns by ensuring they encounter the best practice first
+<!-- rule:109 -->
+- Remove docs content describing features "working as expected" — focus only on integration-specific concerns, limitations, or deviations — Reduces cognitive load and maintenance burden by eliminating noise; prevents documentation staleness from trivial statements
+<!-- rule:67 -->
+- Keep provider-specific config/features in `docs/models/{provider}.md` and `docs/api/models/{provider}.md`; general docs stay provider-agnostic with one minimal example + links — Prevents duplication, keeps general feature docs clean and maintainable, ensures users find provider-specific details in one canonical location rather than scattered across multiple pages
+<!-- rule:941 -->
+- Avoid `test="skip"` in code examples unless unavoidable (external services, credentials, non-deterministic behavior) — use mocks or fixtures instead — Testable documentation examples prove the code works and prevent docs from drifting out of sync with actual behavior
+<!-- rule:727 -->
+- Link to canonical sources rather than duplicating lists or summaries maintained elsewhere — Prevents docs from becoming outdated when the source of truth changes
+<!-- rule:808 -->
+- Focus docs on user tasks and public APIs, defer implementation details to docstrings — Task-oriented guides help users accomplish goals faster, while keeping advanced/internal details in API reference prevents overwhelming users with complexity when sensible defaults exist
+<!-- rule:301 -->
+- In docs, consolidate examples showing parameter variations into one block with notes — split only for mutually exclusive params or distinct use cases — Reduces cognitive load and makes docs more scannable by avoiding repetitive boilerplate for simple parameter alternatives
+<!-- rule:58 -->
+- In docs examples, demonstrate realistic use cases that show *why* the feature matters — prevents misleading users with toy scenarios or debugging code that obscure actual value — Well-crafted examples help users understand when to apply features and avoid implementing unnecessary patterns for problems solvable with simpler approaches
+<!-- rule:54 -->
+- Use fence-level `{test="skip" lint="skip"}` instead of inline suppressions in doc examples — keeps code clean and reader-focused — Documentation code should model best practices; fence-level skip directives separate tooling concerns from the example itself, while inline `# noqa` or `# type: ignore` pollutes pedagogical code with implementation details
+<!-- rule:151 -->
+- Cross-reference alternatives and explain trade-offs when documenting overlapping features — Prevents users from missing better-suited options or implementing duplicate functionality when multiple approaches exist (e.g., `UsageLimits` vs rate-limiting, provider-specific implementations)
+<!-- rule:1112 -->
+- Document default behavior and use cases for all configurable features — helps users decide when to override defaults — Users can't make informed configuration choices without knowing what happens by default and when alternatives are appropriate
+<!-- rule:135 -->
+- Use actual, currently available model names in documentation examples — prevents user confusion and copy-paste errors with non-existent models — Ensures users can run documentation examples without modification and avoids frustration from referencing models that don't exist yet or are hypothetical
+<!-- rule:508 -->
+- Verify all doc links with `make docs-serve` before committing — catches broken internal/external references early — Prevents documentation drift and broken links from reaching users, especially after code refactoring
+<!-- rule:283 -->
+- Use MkDocs admonitions (`!!! note`, `!!! warning`) for callouts, not blockquotes (`>`) or GitHub alerts (`> [!NOTE]`) — Ensures consistent rendering in MkDocs and prevents callouts from cluttering the table of contents
+<!-- rule:618 -->
+- Nest subtopics, examples, and config details within parent sections — improves discoverability and reduces redundant context — Hierarchical organization makes documentation easier to navigate and understand by grouping related content together rather than scattering it across top-level sections or separate files.
+<!-- rule:298 -->
+- In provider feature support tables, use a `Notes` or `Provider Support Notes` column for variations, limitations, and special values — keeps table structure clean and constraints discoverable — Centralizes provider-specific exceptions in one scannable location instead of scattering them across config examples or inline parentheticals, making cross-provider differences easier to find
+<!-- rule:634 -->
+- In provider feature tables, use standard labels (`Full feature support`, `Limited parameter support`) and move unsupported variants to `Unsupported` column, not inline exceptions — Ensures consistent, scannable documentation structure where users can quickly identify exact support boundaries across providers
+<!-- rule:168 -->
+- When documenting alternative approaches, explain tradeoffs (limitations, requirements, benefits, use-cases) and warn about conflicts when combining them — Helps users make informed decisions and avoid subtle bugs from conflicting configurations
+
+<!-- /braindump -->

docs/CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

docs/agent-spec.md

@@ -0,0 +1,188 @@
+# Agent Specs
+
+Agent specs let you define agents declaratively in YAML or JSON — [model](models/overview.md), [instructions](agent.md#instructions), [capabilities](capabilities.md), and all. One line to load, no Python agent construction code required.
+
+This is useful for:
+
+- Separating agent configuration from application code
+- Letting non-developers (prompt engineers, domain experts) configure agents
+- Storing agent definitions alongside other config files
+- Sharing agent configurations across teams or projects
+
+## Defining a spec
+
+A spec file defines the agent's configuration in YAML or JSON:
+
+```yaml {title="agent.yaml" test="skip"}
+model: anthropic:claude-opus-4-6
+instructions: You are a helpful research assistant.
+model_settings:
+  max_tokens: 8192
+capabilities:
+  - WebSearch:
+      local: duckduckgo
+  - Thinking:
+      effort: high
+```
+
+## Loading specs
+
+[`Agent.from_file`][pydantic_ai.Agent.from_file] loads a spec from a YAML or JSON file and constructs an agent:
+
+```python {title="from_file_example.py" test="skip"}
+from pydantic_ai import Agent
+
+agent = Agent.from_file('agent.yaml')
+```
+
+[`Agent.from_spec`][pydantic_ai.Agent.from_spec] accepts a dict or [`AgentSpec`][pydantic_ai.agent.spec.AgentSpec] instance and supports additional keyword arguments that supplement or override the spec:
+
+```python {title="from_spec_example.py"}
+from dataclasses import dataclass
+
+from pydantic_ai import Agent
+
+
+@dataclass
+class UserContext:
+    user_name: str
+
+
+agent = Agent.from_spec(
+    {
+        'model': 'anthropic:claude-opus-4-6',
+        'instructions': 'You are helping {{user_name}}.',
+        'capabilities': [{'WebSearch': {'local': 'duckduckgo'}}],
+    },
+    deps_type=UserContext,
+)
+```
+
+Keyword arguments interact with spec fields as follows:
+
+* **Scalar fields** (`model`, `name`, `end_strategy`, etc.) — the keyword argument overrides the spec value when provided. For retry budgets, the `retries` keyword argument overrides the spec's `retries` value.
+* **`instructions`** — merged: spec instructions come first, then keyword argument instructions.
+* **`capabilities`** — merged: spec capabilities come first, then keyword argument capabilities.
+* **`model_settings`** — merged additively: keyword argument settings override matching spec settings.
+* **`output_type`** — takes precedence over `output_schema` from the spec.
+
+When `deps_type` is passed, [template strings](#template-strings) in the spec's `instructions`, `description`, and capability arguments are compiled and validated against the deps type at construction time.
+
+For more control over spec loading, use [`AgentSpec.from_file`][pydantic_ai.agent.spec.AgentSpec.from_file] to load the spec separately before passing it to `Agent.from_spec`.
+
+## Template strings
+
+[`TemplateStr`][pydantic_ai.TemplateStr] provides Handlebars-style templates (`{{variable}}`) that are rendered against the agent's [dependencies](dependencies.md) at runtime. In spec files, strings containing `{{` are automatically converted to template strings:
+
+```yaml {test="skip"}
+instructions: "You are assisting {{name}}, who is a {{role}}."
+```
+
+Template variables are resolved from the fields of the `deps` object. When a `deps_type` (or [`deps_schema`](#deps_schema)) is provided, template variable names are validated at construction time.
+
+In Python code, [`TemplateStr`][pydantic_ai.TemplateStr] can be used explicitly, but a callable with [`RunContext`][pydantic_ai.tools.RunContext] is generally preferred for IDE autocomplete and type checking:
+
+```python {title="template_instructions.py"}
+from dataclasses import dataclass
+
+from pydantic_ai import Agent, TemplateStr
+
+
+@dataclass
+class UserProfile:
+    name: str
+    role: str
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    deps_type=UserProfile,
+    instructions=TemplateStr('You are assisting {{name}}, who is a {{role}}.'),
+)
+result = agent.run_sync('hello', deps=UserProfile(name='Alice', role='engineer'))
+print(result.output)
+#> Hello! How can I help you today?
+```
+
+## Capability spec syntax
+
+Capabilities in specs support three forms:
+
+* `'MyCapability'` — no arguments, calls `MyCapability.from_spec()`
+* `{'MyCapability': value}` — single positional argument, calls `MyCapability.from_spec(value)`
+* `{'MyCapability': {key: value, ...}}` — keyword arguments, calls `MyCapability.from_spec(**kwargs)`
+
+## Custom capabilities in specs
+
+See [Publishing capabilities](capabilities.md#publishing-capabilities) for how to make custom capabilities work with agent specs.
+
+## `AgentSpec` reference
+
+The [`AgentSpec`][pydantic_ai.agent.spec.AgentSpec] model represents the full spec structure:
+
+| Field | Type | Description |
+|---|---|---|
+| `model` | `str` | [Model](models/overview.md) name (required) |
+| `name` | `str \| None` | Agent name |
+| `description` | `str \| None` | Agent description (supports [templates](#template-strings)) |
+| `instructions` | `str \| list[str] \| None` | [Instructions](agent.md#instructions) (supports [templates](#template-strings)) |
+| `model_settings` | `dict \| None` | [Model settings](agent.md#model-run-settings) |
+| `capabilities` | `list` | [Capabilities](capabilities.md) (see [spec syntax](#capability-spec-syntax)) |
+| `deps_schema` | `dict \| None` | JSON Schema for [template string](#template-strings) validation (see below) |
+| `output_schema` | `dict \| None` | JSON Schema for [structured output](output.md) (see below) |
+| `retries` | `int \| AgentRetries \| None` | Retry budgets for [tools](tools-advanced.md#tool-retries) and [output validation](output.md#output-validator-functions). Pass an integer to use the same budget for both, or [`AgentRetries`][pydantic_ai.agent.AgentRetries] to configure them separately. |
+| `end_strategy` | `EndStrategy` | When to stop (`'early'`, `'graceful'`, or `'exhaustive'`) |
+| `tool_timeout` | `float \| None` | Default [tool](tools.md) timeout in seconds |
+| `instrument` | `bool \| None` | Enable [Logfire](logfire.md) instrumentation |
+| `metadata` | `dict \| None` | Agent [metadata](agent.md#run-metadata) |
+
+### `deps_schema`
+
+When loading a spec file without a Python `deps_type`, `deps_schema` provides a JSON Schema that validates [template string](#template-strings) variable names at construction time. It does **not** validate the actual deps object at runtime — it only ensures that template variables like `{{user_name}}` correspond to properties defined in the schema.
+
+### `output_schema`
+
+When provided (and no `output_type` keyword argument is passed to `from_spec`), `output_schema` defines the structure the model should produce as its final output. Under the hood, it creates a [`StructuredDict`][pydantic_ai.output.StructuredDict] output type: the JSON Schema is sent to the model API so the model knows what structure to produce, and the response is returned as a `dict[str, Any]`.
+
+!!! note
+    The model's response is not validated against the schema's `properties` or `required` fields — it is accepted as a plain dict. The schema serves as an instruction to the model, not a runtime validation constraint.
+
+```yaml {title="agent_with_schema.yaml" test="skip"}
+model: anthropic:claude-opus-4-6
+deps_schema:
+  type: object
+  properties:
+    user_name:
+      type: string
+  required: [user_name]
+output_schema:
+  type: object
+  properties:
+    answer:
+      type: string
+    confidence:
+      type: number
+  required: [answer, confidence]
+instructions: "You are helping {{user_name}}. Always include a confidence score."
+capabilities:
+  - WebSearch:
+      local: duckduckgo
+```
+
+## Saving specs
+
+[`AgentSpec.to_file`][pydantic_ai.agent.spec.AgentSpec.to_file] saves a spec to YAML or JSON and optionally generates a companion JSON Schema file for editor autocompletion:
+
+```python {title="save_spec_example.py"}
+from pydantic_ai import AgentSpec
+
+spec = AgentSpec(
+    model='anthropic:claude-opus-4-6',
+    instructions='You are a helpful assistant.',
+    capabilities=[{'WebSearch': {'local': 'duckduckgo'}}],
+)
+spec.to_file('agent.yaml')
+# Also generates ./agent_schema.json for editor autocompletion
+```
+
+The generated JSON Schema file enables autocompletion and validation in editors that support the [YAML Language Server](https://github.com/redhat-developer/yaml-language-server) protocol. Pass `schema_path=None` to skip schema generation.

docs/api/agent.md

@@ -0,0 +1,16 @@
+# `pydantic_ai.agent`
+
+::: pydantic_ai.agent
+    options:
+        members:
+            - Agent
+            - AbstractAgent
+            - WrapperAgent
+            - AgentRetries
+            - AgentRun
+            - AgentRunResult
+            - EndStrategy
+            - RunOutputDataT
+            - capture_run_messages
+            - InstrumentationSettings
+            - EventStreamHandler

docs/api/capabilities.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.capabilities`
+
+::: pydantic_ai.capabilities

docs/api/common_tools.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.common_tools`
+
+::: pydantic_ai.common_tools.duckduckgo
+
+::: pydantic_ai.common_tools.exa
+
+::: pydantic_ai.common_tools.tavily

docs/api/concurrency.md

@@ -0,0 +1,15 @@
+# `pydantic_ai` — Concurrency
+
+::: pydantic_ai.ConcurrencyLimitedModel
+
+::: pydantic_ai.limit_model_concurrency
+
+::: pydantic_ai.AbstractConcurrencyLimiter
+
+::: pydantic_ai.ConcurrencyLimiter
+
+::: pydantic_ai.ConcurrencyLimit
+
+::: pydantic_ai.AnyConcurrencyLimit
+
+::: pydantic_ai.ConcurrencyLimitExceeded

docs/api/direct.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.direct`
+
+::: pydantic_ai.direct

docs/api/durable_exec.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.durable_exec`
+
+::: pydantic_ai.durable_exec.temporal
+
+::: pydantic_ai.durable_exec.dbos
+
+::: pydantic_ai.durable_exec.prefect

docs/api/embeddings.md

@@ -0,0 +1,27 @@
+# `pydantic_ai.embeddings`
+
+::: pydantic_ai.embeddings
+
+::: pydantic_ai.embeddings.base
+
+::: pydantic_ai.embeddings.result
+
+::: pydantic_ai.embeddings.settings
+
+::: pydantic_ai.embeddings.openai
+
+::: pydantic_ai.embeddings.cohere
+
+::: pydantic_ai.embeddings.google
+
+::: pydantic_ai.embeddings.bedrock
+
+::: pydantic_ai.embeddings.voyageai
+
+::: pydantic_ai.embeddings.sentence_transformers
+
+::: pydantic_ai.embeddings.test
+
+::: pydantic_ai.embeddings.wrapper
+
+::: pydantic_ai.embeddings.instrumented

docs/api/exceptions.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.exceptions`
+
+::: pydantic_ai.exceptions

docs/api/ext.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.ext`
+
+::: pydantic_ai.ext.langchain

docs/api/format_prompt.md

@@ -0,0 +1,6 @@
+# `pydantic_ai.format_prompt`
+
+::: pydantic_ai.format_prompt
+    options:
+        members:
+            - format_as_xml

docs/api/function_signature.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.function_signature`
+
+::: pydantic_ai.function_signature

docs/api/mcp.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.mcp`
+
+::: pydantic_ai.mcp

docs/api/messages.md

@@ -0,0 +1,20 @@
+# `pydantic_ai.messages`
+
+The structure of [`ModelMessage`][pydantic_ai.messages.ModelMessage] can be shown as a graph:
+
+```mermaid
+graph RL
+    SystemPromptPart(SystemPromptPart) --- ModelRequestPart
+    UserPromptPart(UserPromptPart) --- ModelRequestPart
+    ToolReturnPart(ToolReturnPart) --- ModelRequestPart
+    RetryPromptPart(RetryPromptPart) --- ModelRequestPart
+    TextPart(TextPart) --- ModelResponsePart
+    ToolCallPart(ToolCallPart) --- ModelResponsePart
+    ThinkingPart(ThinkingPart) --- ModelResponsePart
+    ModelRequestPart("ModelRequestPart<br>(Union)") --- ModelRequest
+    ModelRequest("ModelRequest(parts=list[...])") --- ModelMessage
+    ModelResponsePart("ModelResponsePart<br>(Union)") --- ModelResponse
+    ModelResponse("ModelResponse(parts=list[...])") --- ModelMessage("ModelMessage<br>(Union)")
+```
+
+::: pydantic_ai.messages

docs/api/models/anthropic.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.anthropic`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Anthropic](../../models/anthropic.md).
+
+::: pydantic_ai.models.anthropic

docs/api/models/base.md

@@ -0,0 +1,14 @@
+# `pydantic_ai.models`
+
+::: pydantic_ai.models
+    options:
+      members:
+        - KnownModelName
+        - known_model_names
+        - ModelRequestParameters
+        - Model
+        - AbstractToolDefinition
+        - StreamedResponse
+        - ALLOW_MODEL_REQUESTS
+        - check_allow_model_requests
+        - override_allow_model_requests

docs/api/models/bedrock.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.bedrock`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Bedrock](../../models/bedrock.md).
+
+::: pydantic_ai.models.bedrock

docs/api/models/cerebras.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.cerebras`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Cerebras](../../models/cerebras.md).
+
+::: pydantic_ai.models.cerebras

docs/api/models/cohere.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.cohere`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Cohere](../../models/cohere.md).
+
+::: pydantic_ai.models.cohere

docs/api/models/fallback.md

@@ -0,0 +1,3 @@
+# pydantic_ai.models.fallback
+
+::: pydantic_ai.models.fallback

docs/api/models/function.md

@@ -0,0 +1,64 @@
+# `pydantic_ai.models.function`
+
+A model controlled by a local function.
+
+[`FunctionModel`][pydantic_ai.models.function.FunctionModel] is similar to [`TestModel`](test.md),
+but allows greater control over the model's behavior.
+
+Its primary use case is for more advanced unit testing than is possible with `TestModel`.
+
+Here's a minimal example:
+
+```py {title="function_model_usage.py" call_name="test_my_agent" noqa="I001"}
+from pydantic_ai import Agent
+from pydantic_ai import ModelMessage, ModelResponse, TextPart
+from pydantic_ai.models.function import FunctionModel, AgentInfo
+
+my_agent = Agent('openai:gpt-5.2')
+
+
+async def model_function(
+    messages: list[ModelMessage], info: AgentInfo
+) -> ModelResponse:
+    print(messages)
+    """
+    [
+        ModelRequest(
+            parts=[
+                UserPromptPart(
+                    content='Testing my agent...',
+                    timestamp=datetime.datetime(...),
+                )
+            ],
+            timestamp=datetime.datetime(...),
+            run_id='...',
+            conversation_id='...',
+        )
+    ]
+    """
+    print(info)
+    """
+    AgentInfo(
+        function_tools=[],
+        allow_text_output=True,
+        output_tools=[],
+        model_settings=None,
+        model_request_parameters=ModelRequestParameters(
+            function_tools=[], native_tools=[], output_tools=[]
+        ),
+        instructions=None,
+    )
+    """
+    return ModelResponse(parts=[TextPart('hello world')])
+
+
+async def test_my_agent():
+    """Unit test for my_agent, to be run by pytest."""
+    with my_agent.override(model=FunctionModel(model_function)):
+        result = await my_agent.run('Testing my agent...')
+        assert result.output == 'hello world'
+```
+
+See [Unit testing with `FunctionModel`](../../testing.md#unit-testing-with-functionmodel) for detailed documentation.
+
+::: pydantic_ai.models.function

docs/api/models/google.md

@@ -0,0 +1,10 @@
+# `pydantic_ai.models.google`
+
+Interface that uses the [`google-genai`](https://pypi.org/project/google-genai/) package under the hood to
+access Google's Gemini models via both the Gemini API and Google Cloud (formerly known as Vertex AI).
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Google](../../models/google.md).
+
+::: pydantic_ai.models.google

docs/api/models/groq.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.groq`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Groq](../../models/groq.md).
+
+::: pydantic_ai.models.groq

docs/api/models/huggingface.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.huggingface`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Hugging Face](../../models/huggingface.md).
+
+::: pydantic_ai.models.huggingface

docs/api/models/instrumented.md

@@ -0,0 +1,3 @@
+# pydantic_ai.models.instrumented
+
+::: pydantic_ai.models.instrumented

docs/api/models/mcp-sampling.md

@@ -0,0 +1,3 @@
+# pydantic_ai.models.mcp_sampling
+
+::: pydantic_ai.models.mcp_sampling

docs/api/models/mistral.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.mistral`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Mistral](../../models/mistral.md).
+
+::: pydantic_ai.models.mistral

docs/api/models/ollama.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.ollama`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Ollama](../../models/ollama.md).
+
+::: pydantic_ai.models.ollama

docs/api/models/openai.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.openai`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for OpenAI](../../models/openai.md).
+
+::: pydantic_ai.models.openai

docs/api/models/openrouter.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.openrouter`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for OpenRouter](../../models/openrouter.md).
+
+::: pydantic_ai.models.openrouter

docs/api/models/test.md

@@ -0,0 +1,25 @@
+# `pydantic_ai.models.test`
+
+Utility model for quickly testing apps built with Pydantic AI.
+
+Here's a minimal example:
+
+```py {title="test_model_usage.py" call_name="test_my_agent" noqa="I001"}
+from pydantic_ai import Agent
+from pydantic_ai.models.test import TestModel
+
+my_agent = Agent('openai:gpt-5.2', instructions='...')
+
+
+async def test_my_agent():
+    """Unit test for my_agent, to be run by pytest."""
+    m = TestModel()
+    with my_agent.override(model=m):
+        result = await my_agent.run('Testing my agent...')
+        assert result.output == 'success (no tool calls)'
+    assert m.last_model_request_parameters.function_tools == []
+```
+
+See [Unit testing with `TestModel`](../../testing.md#unit-testing-with-testmodel) for detailed documentation.
+
+::: pydantic_ai.models.test

docs/api/models/wrapper.md

@@ -0,0 +1,3 @@
+# pydantic_ai.models.wrapper
+
+::: pydantic_ai.models.wrapper

docs/api/models/xai.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.xai`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for xAI](../../models/xai.md).
+
+::: pydantic_ai.models.xai

docs/api/models/zai.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.models.zai`
+
+## Setup
+
+For details on how to set up authentication with this model, see [model configuration for Z.AI](../../models/zai.md).
+
+::: pydantic_ai.models.zai

docs/api/native_tools.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.native_tools`
+
+::: pydantic_ai.native_tools

docs/api/output.md

@@ -0,0 +1,13 @@
+# `pydantic_ai.output`
+
+::: pydantic_ai.output
+    options:
+        inherited_members: true
+        members:
+            - OutputDataT
+            - ToolOutput
+            - NativeOutput
+            - PromptedOutput
+            - TextOutput
+            - StructuredDict
+            - DeferredToolRequests

docs/api/profiles.md

@@ -0,0 +1,33 @@
+# `pydantic_ai.profiles`
+
+::: pydantic_ai.profiles
+    options:
+      members:
+        - ModelProfile
+        - ModelProfileSpec
+        - merge_profile
+        - DEFAULT_PROFILE
+        - DEFAULT_PROMPTED_OUTPUT_TEMPLATE
+        - DEFAULT_THINKING_TAGS
+
+::: pydantic_ai.profiles.openai
+
+::: pydantic_ai.profiles.anthropic
+
+::: pydantic_ai.profiles.google
+
+::: pydantic_ai.profiles.meta
+
+::: pydantic_ai.profiles.amazon
+
+::: pydantic_ai.profiles.deepseek
+
+::: pydantic_ai.profiles.grok
+
+::: pydantic_ai.profiles.mistral
+
+::: pydantic_ai.profiles.qwen
+
+::: pydantic_ai.profiles.groq
+
+::: pydantic_ai.profiles.zai

docs/api/providers.md

@@ -0,0 +1,59 @@
+# `pydantic_ai.providers`
+
+::: pydantic_ai.providers.Provider
+
+::: pydantic_ai.providers.gateway.gateway_provider
+
+::: pydantic_ai.providers.anthropic.AnthropicProvider
+
+::: pydantic_ai.providers.google
+
+::: pydantic_ai.providers.openai
+
+::: pydantic_ai.providers.xai
+
+::: pydantic_ai.providers.deepseek
+
+::: pydantic_ai.providers.bedrock
+
+::: pydantic_ai.providers.groq
+
+::: pydantic_ai.providers.azure
+
+::: pydantic_ai.providers.cohere
+
+::: pydantic_ai.providers.voyageai.VoyageAIProvider
+
+::: pydantic_ai.providers.cerebras.CerebrasProvider
+
+::: pydantic_ai.providers.mistral.MistralProvider
+
+::: pydantic_ai.providers.fireworks.FireworksProvider
+
+::: pydantic_ai.providers.together.TogetherProvider
+
+::: pydantic_ai.providers.heroku.HerokuProvider
+
+::: pydantic_ai.providers.github.GitHubProvider
+
+::: pydantic_ai.providers.openrouter.OpenRouterProvider
+
+::: pydantic_ai.providers.vercel.VercelProvider
+
+::: pydantic_ai.providers.huggingface.HuggingFaceProvider
+
+::: pydantic_ai.providers.moonshotai.MoonshotAIProvider
+
+::: pydantic_ai.providers.ollama.OllamaProvider
+
+::: pydantic_ai.providers.litellm.LiteLLMProvider
+
+::: pydantic_ai.providers.nebius.NebiusProvider
+
+::: pydantic_ai.providers.ovhcloud.OVHcloudProvider
+
+::: pydantic_ai.providers.alibaba.AlibabaProvider
+
+::: pydantic_ai.providers.sambanova.SambaNovaProvider
+
+::: pydantic_ai.providers.zai.ZaiProvider

docs/api/pydantic_evals/dataset.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.dataset`
+
+::: pydantic_evals.dataset

docs/api/pydantic_evals/evaluators.md

@@ -0,0 +1,5 @@
+# `pydantic_evals.evaluators`
+
+::: pydantic_evals.evaluators
+
+::: pydantic_evals.evaluators.llm_as_a_judge

docs/api/pydantic_evals/generation.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.generation`
+
+::: pydantic_evals.generation

docs/api/pydantic_evals/lifecycle.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.lifecycle`
+
+::: pydantic_evals.lifecycle

docs/api/pydantic_evals/online.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.online`
+
+::: pydantic_evals.online

docs/api/pydantic_evals/online_capability.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.online_capability`
+
+::: pydantic_evals.online_capability

docs/api/pydantic_evals/otel.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.otel`
+
+::: pydantic_evals.otel

docs/api/pydantic_evals/reporting.md

@@ -0,0 +1,3 @@
+# `pydantic_evals.reporting`
+
+::: pydantic_evals.reporting

docs/api/pydantic_graph/basenode.md

@@ -0,0 +1,13 @@
+# `pydantic_graph.basenode`
+
+::: pydantic_graph.basenode
+    options:
+        members:
+            - StateT
+            - GraphRunContext
+            - BaseNode
+            - End
+            - Edge
+            - DepsT
+            - RunEndT
+            - NodeRunEndT

docs/api/pydantic_graph/decision.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.decision`
+
+::: pydantic_graph.decision

docs/api/pydantic_graph/exceptions.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.exceptions`
+
+::: pydantic_graph.exceptions

docs/api/pydantic_graph/graph_builder.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.graph_builder`
+
+::: pydantic_graph.graph_builder

docs/api/pydantic_graph/join.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.join`
+
+::: pydantic_graph.join

docs/api/pydantic_graph/node.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.node`
+
+::: pydantic_graph.node

docs/api/pydantic_graph/step.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.step`
+
+::: pydantic_graph.step

docs/api/pydantic_graph/util.md

@@ -0,0 +1,3 @@
+# `pydantic_graph.util`
+
+::: pydantic_graph.util

docs/api/result.md

@@ -0,0 +1,8 @@
+# `pydantic_ai.result`
+
+::: pydantic_ai.result
+    options:
+        inherited_members: true
+        members:
+            - StreamedRunResult
+            - StreamedRunResultSync

docs/api/retries.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.retries`
+
+::: pydantic_ai.retries

docs/api/run.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.run`
+
+::: pydantic_ai.run

docs/api/settings.md

@@ -0,0 +1,8 @@
+# `pydantic_ai.settings`
+
+::: pydantic_ai.settings
+    options:
+      inherited_members: true
+      members:
+        - ModelSettings
+        - ToolOrOutput

docs/api/tools.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.tools`
+
+::: pydantic_ai.tools

docs/api/toolsets.md

@@ -0,0 +1,19 @@
+# `pydantic_ai.toolsets`
+
+::: pydantic_ai.toolsets
+    options:
+        members:
+        - AbstractToolset
+        - CombinedToolset
+        - ExternalToolset
+        - ApprovalRequiredToolset
+        - FilteredToolset
+        - FunctionToolset
+        - IncludeReturnSchemasToolset
+        - DeferredLoadingToolset
+        - PrefixedToolset
+        - RenamedToolset
+        - SetMetadataToolset
+        - PreparedToolset
+        - WrapperToolset
+        - ToolsetFunc

docs/api/ui/ag_ui.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.ui.ag_ui`
+
+::: pydantic_ai.ui.ag_ui

docs/api/ui/base.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.ui`
+
+::: pydantic_ai.ui

docs/api/ui/vercel_ai.md

@@ -0,0 +1,7 @@
+# `pydantic_ai.ui.vercel_ai`
+
+::: pydantic_ai.ui.vercel_ai
+
+::: pydantic_ai.ui.vercel_ai.request_types
+
+::: pydantic_ai.ui.vercel_ai.response_types

docs/api/usage.md

@@ -0,0 +1,3 @@
+# `pydantic_ai.usage`
+
+::: pydantic_ai.usage

docs/changelog.md

@@ -0,0 +1,332 @@
+# Upgrade Guide
+
+In September 2025, Pydantic AI reached V1 and committed to API stability: no changes that break your code until V2. V2 is now available, collecting the breaking and behavior changes that stability guarantee didn't allow. This guide is the canonical place to learn what's in V2, how to install it, and how to upgrade; for the guarantees behind these version numbers, see the [Version Policy](version-policy.md).
+
+## Breaking Changes
+
+Here's a filtered list of the breaking changes for each version to help you upgrade Pydantic AI.
+
+### v2.0.0 (2026-06-23)
+
+The stable V2.0 release. There are no new breaking or behavior changes since the betas; the full breaking-change list and recommended upgrade path are in the [v2.0.0b1](#v200b1-2026-05-20) entry below. Install it with:
+
+```bash
+uv add pydantic-ai
+```
+
+### v2.0.0b7 (2026-06-10)
+
+The seventh V2 beta, forked from **v1.107.0**. There are no new V2 breaking or behavior changes since [v2.0.0b6](#v200b6-2026-06-04) below — everything in that entry applies unchanged — but this beta picks up the latest V1 release on top, which adds Claude Fable 5 / Mythos 5 model support and OpenRouter prompt caching (`CachePoint`), plus `known_model_names()` and Anthropic fixes; see the [v1.107.0 release notes](https://github.com/pydantic/pydantic-ai/releases/tag/v1.107.0) for the full list.
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b7"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.107.0**.
+
+### v2.0.0b6 (2026-06-04)
+
+The sixth V2 beta, forked from **v1.106.0**. There are no new V2 breaking or behavior changes since [v2.0.0b5](#v200b5-2026-06-02) below — everything in that entry applies unchanged — but this beta picks up the latest V1 release on top, which adds `api_host`/`timeout` configuration and base `seed` mapping for the xAI provider, plus streaming and data-URI handling fixes; see the [v1.106.0 release notes](https://github.com/pydantic/pydantic-ai/releases/tag/v1.106.0) for the full list.
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b6"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.106.0**.
+
+### v2.0.0b5 (2026-06-02)
+
+The fifth V2 beta, forked from **v1.105.0**. There are no new V2 breaking or behavior changes since [v2.0.0b4](#v200b4-2026-05-28) below — everything in that entry (including the prepare-callbacks change) still applies — but this beta picks up the latest V1 release on top, which adds [on-demand (deferred-loading) capabilities](https://github.com/pydantic/pydantic-ai/pull/5230) and [Grok 4.3 `reasoning_effort` support](https://github.com/pydantic/pydantic-ai/pull/5454), plus `GoogleModelSettings.google_cached_content` and Temporal `gateway/` fixes; see the [v1.105.0 release notes](https://github.com/pydantic/pydantic-ai/releases/tag/v1.105.0) for the full list.
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b5"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.105.0**.
+
+### v2.0.0b4 (2026-05-28)
+
+The fourth V2 beta, forked from **v1.104.0**. One new V2 behavior change since [v2.0.0b3](#v200b3-2026-05-22):
+
+- Prepare callbacks (`prepare_tools=` / `PrepareTools` capability) that return `None` now raise `TypeError` instead of silently stripping all tools. V1.103.0 announces this change via `PydanticAIDeprecationWarning` (see [#5188](https://github.com/pydantic/pydantic-ai/pull/5188)); V2 turns the warning into a hard error (see [#5668](https://github.com/pydantic/pydantic-ai/pull/5668)). Return an empty list (`[]`) when you mean "no tools for this turn."
+
+This beta also picks up two V1 releases on top — [v1.103.0](https://github.com/pydantic/pydantic-ai/releases/tag/v1.103.0) and [v1.104.0](https://github.com/pydantic/pydantic-ai/releases/tag/v1.104.0) — together adding Claude Opus 4.8 support, `McpServer.list_prompts` / `get_prompt`, message-timestamp roundtripping through `VercelAIAdapter`'s `UIMessage.metadata`, OpenRouter eager input streaming, and several Bedrock and UI fixes. See those release notes for the full list.
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b4"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.104.0**.
+
+### v2.0.0b3 (2026-05-22)
+
+The third V2 beta, forked from **v1.102.0**. There are no new V2 breaking changes since [v2.0.0b1](#v200b1-2026-05-20) below — everything in that entry applies unchanged — but this beta picks up the latest V1 release on top, which is a bug-fix release; see the [v1.102.0 release notes](https://github.com/pydantic/pydantic-ai/releases/tag/v1.102.0) for the full list.
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b3"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.102.0**.
+
+### v2.0.0b2 (2026-05-21)
+
+The second V2 beta, forked from **v1.101.0**. There are no new V2 breaking changes since [v2.0.0b1](#v200b1-2026-05-20) below — everything in that entry applies unchanged — but this beta picks up the latest V1 release on top, which adds the [pending message queue](https://github.com/pydantic/pydantic-ai/pull/4980) (`ctx.enqueue` / `agent_run.enqueue`).
+
+Install it the same way, pinning the exact pre-release version:
+
+```bash
+pip/uv-add "pydantic-ai==2.0.0b2"
+```
+
+For the full breaking-change list and the recommended upgrade path, see the [v2.0.0b1](#v200b1-2026-05-20) entry below; the only difference is that the latest V1 to upgrade through first is now **v1.101.0**.
+
+### v2.0.0b1 (2026-05-20)
+
+The first V2 beta, forked from **v1.100.0**, which deprecates most of what V2 removes. V2 leans into a harness-first design with [capabilities](capabilities.md) as a core primitive: a single, composable unit that bundles an agent's tools, [hooks](hooks.md), instructions, and model settings, reaching every layer of the agent through one concept. Many of V2's changes move configuration that used to be spread across `Agent` arguments onto that primitive, alongside the behavior changes that V1's stability guarantee didn't allow. Pydantic AI stays a small core: some capabilities ship with it, more come from the first-party [Pydantic AI Harness](harness/overview.md), and others are third-party or your own.
+
+The breaking changes below are split into two groups:
+
+- [**Changes not covered by deprecation warnings**](#changes-not-covered-by-deprecation-warnings) — removals and behavior changes that couldn't be announced via a V1 deprecation warning. Review these even if you're already on the latest V1 with no warnings.
+- [**Changes covered by deprecation warnings**](#changes-covered-by-deprecation-warnings) — if you upgraded to the latest V1 and resolved every deprecation warning, you've already made these. They're listed with full before → after for reference.
+
+**Recommended upgrade path.** To make the jump as smooth as possible:
+
+1. **Upgrade to the latest V1 release.** Most of what V2 removes is deprecated as of **v1.100.0** (the release this beta is forked from), so any V1 at or above that version surfaces those warnings.
+2. **Resolve every deprecation warning.** The [changes covered by deprecation warnings](#changes-covered-by-deprecation-warnings) were announced in V1 via warnings that name the new API and, where possible, include a migration snippet. Run your test suite (or app) with warnings visible and address each one — by hand or by pointing a coding agent at them — to migrate across the bulk of V2 ahead of time.
+3. **Upgrade to V2** and make the [changes not covered by deprecation warnings](#changes-not-covered-by-deprecation-warnings) — primarily default-behavior changes and a handful of removals with no V1 deprecation.
+
+You can also upgrade straight to V2 and work through the list below directly — it's organized so a coding agent can apply the code changes mechanically. Resolving deprecation warnings on the latest V1 first is still the smoother path, since it spreads the work out and leaves you only the behavior changes to reason about consciously at the end.
+
+Message history serialized with V1 (via [`ModelMessagesTypeAdapter`][pydantic_ai.messages.ModelMessagesTypeAdapter]) continues to deserialize in V2.
+
+#### Changes not covered by deprecation warnings
+
+These removals and behavior changes could not be announced via a V1 deprecation warning, so review them even if you've resolved every deprecation warning on the latest V1.
+
+**Code changes:**
+
+- Generic type parameter defaults changed from `None` to `object`: an un-parameterized `Agent(...)` now infers `Agent[object, str]` instead of `Agent[None, str]`, and the `pydantic_graph` `StateT`/`RunEndT`/`DepsT` defaults changed to match. Update explicit `Agent[None, ...]`, `RunContext[None]`, and `Tool[None]` annotations that don't actually require `None` dependencies to use `object`. This is a type-checking-only change; runtime behavior is unchanged. See [#5307](https://github.com/pydantic/pydantic-ai/pull/5307).
+- The `pydantic_graph.persistence` package and the `pydantic_graph.mermaid` module are removed, with no V2 equivalent for graph state persistence or standalone Mermaid generation (render diagrams with `Graph.render()`). The move of the [`GraphBuilder`][pydantic_graph.GraphBuilder] API out of `pydantic_graph.beta` to the top-level `pydantic_graph` *was* deprecation-announced; see [below](#changes-covered-by-deprecation-warnings). See [#5470](https://github.com/pydantic/pydantic-ai/pull/5470).
+- [`ModelProfile`][pydantic_ai.profiles.ModelProfile] and its subclasses are now `TypedDict`s instead of dataclasses. Passing `profile=OpenAIModelProfile(field=value)` into a model still works unchanged; the migration only matters if you read or mutate profile fields, or call `.update()`/`.from_profile()`. See [`ModelProfile` is now a `TypedDict`](#modelprofile-is-now-a-typeddict) below. ([#5481](https://github.com/pydantic/pydantic-ai/pull/5481))
+
+**Default behavior changes** — same API, different runtime behavior (roughly ordered by how many users they affect):
+
+- A bare `uv add pydantic-ai` / `pip install pydantic-ai` now installs a slimmer set of extras (frontier providers plus minimal integrations); providers like `bedrock`, `groq`, and `mistral` are no longer included by default, so you'll need to add the extras you use. See [Slimmer default extras](#slimmer-default-pydantic-ai-extras) below. ([#5467](https://github.com/pydantic/pydantic-ai/pull/5467))
+- The default `end_strategy` changed from `'early'` to `'graceful'`: when a model calls function tools in the same response as a successful output tool, those function tools now run (and their side effects happen) instead of being skipped, and tool calls run in the order the model emitted them. See [Parallel tool-call execution order](#parallel-tool-call-execution-runs-in-emission-order) below. ([#5339](https://github.com/pydantic/pydantic-ai/pull/5339))
+- The default instrumentation format is now version 5, and agent run spans report token usage under `gen_ai.aggregated_usage.*`. See [Instrumentation defaults](#instrumentation-defaults-to-version-5-with-aggregated-usage-attributes) below. ([#5523](https://github.com/pydantic/pydantic-ai/pull/5523))
+- [`capture_run_messages()`][pydantic_ai.capture_run_messages] now also captures the partial `ModelRequest`/`ModelResponse` from an interrupted run, marked with `state='interrupted'` (a new `ModelRequest.state` field is added). Code that asserts on exact captured-message counts on error paths may need updating. See [#5364](https://github.com/pydantic/pydantic-ai/pull/5364).
+- Output tool calls and returns now emit dedicated `OutputToolCallEvent`/`OutputToolResultEvent` instead of `FunctionToolCallEvent`/`FunctionToolResultEvent`. Separately, native tool calls and returns no longer emit dedicated events at all — the `BuiltinToolCallEvent`/`BuiltinToolResultEvent` classes are removed and they surface only via the standard `PartStartEvent`/`PartDeltaEvent`. See [#5332](https://github.com/pydantic/pydantic-ai/pull/5332) and [#5476](https://github.com/pydantic/pydantic-ai/pull/5476).
+
+##### [`ModelProfile`][pydantic_ai.profiles.ModelProfile] is now a `TypedDict`
+
+See the [Model Profile guide](models/openai.md#model-profile) for an overview of what a model profile is and how to configure one.
+
+[`ModelProfile`][pydantic_ai.profiles.ModelProfile] and all its subclasses ([`OpenAIModelProfile`][pydantic_ai.profiles.openai.OpenAIModelProfile], [`AnthropicModelProfile`][pydantic_ai.profiles.anthropic.AnthropicModelProfile], [`GoogleModelProfile`][pydantic_ai.profiles.google.GoogleModelProfile], `BedrockModelProfile`, etc.) are now `TypedDict(total=False)` instead of `@dataclass`. This unifies the mental model with [`ModelSettings`][pydantic_ai.settings.ModelSettings] (also a `TypedDict`) and enables direct dict-spread for cross-class merging.
+
+`ModelProfile.update()` and `ModelProfile.from_profile()` are removed; use the module-level [`merge_profile`][pydantic_ai.profiles.merge_profile] (later argument wins per key).
+
+Migration recipes:
+
+| v1 (dataclass) | v2 (TypedDict) |
+|---|---|
+| `OpenAIModelProfile(field=value)` | Same syntax; returns a partial `dict` instead of a fully-defaulted instance. |
+| `profile.field` (attribute read) | `profile.get('field', <default>)` — non-trivial defaults are exported from [`pydantic_ai.profiles`][pydantic_ai.profiles] (e.g. [`DEFAULT_THINKING_TAGS`][pydantic_ai.profiles.DEFAULT_THINKING_TAGS], [`DEFAULT_PROMPTED_OUTPUT_TEMPLATE`][pydantic_ai.profiles.DEFAULT_PROMPTED_OUTPUT_TEMPLATE]); the fully-merged base is [`DEFAULT_PROFILE`][pydantic_ai.profiles.DEFAULT_PROFILE]. |
+| `profile.field = value` (attribute write) | `profile['field'] = value` |
+| `dataclasses.replace(profile, field=value)` | `{**profile, 'field': value}` or `merge_profile(profile, ModelProfile(field=value))` |
+| `profile.update(other)` | `merge_profile(profile, other)` |
+| `OpenAIModelProfile.from_profile(p)` | Just `p` — no upcasting needed |
+| `Model(name, profile=full_profile)` (full replace) | Now merges on top of the provider's default profile — usually what you want. For a hard replace use `Model(name, profile=lambda _default: full_profile)`. |
+| `Model(name, profile=fn)` where `fn: Callable[[str], ModelProfile \| None]` | Removed — the user-passed callable is now `Callable[[ModelProfile], ModelProfile]`, receiving the resolved default and returning the final profile. The `(model_name: str) -> ModelProfile \| None` shape is still accepted internally by `Provider.model_profile`. |
+| `isinstance(profile, OpenAIModelProfile)` | Not supported by `TypedDict` at runtime — raises `TypeError`. Use `isinstance(profile, dict)` or check key presence (`'openai_chat_supports_web_search' in profile`). Pyright still narrows correctly via the TypedDict subclass annotation. |
+
+`Model.profile` is now the single source of truth for the **resolved** profile. It is composed by [`merge_profile`][pydantic_ai.profiles.merge_profile] in this order (later wins):
+
+1. [`DEFAULT_PROFILE`][pydantic_ai.profiles.DEFAULT_PROFILE] — base defaults for every documented key.
+2. `Provider.model_profile(model_name)` — provider/model-specific resolution.
+3. The user's `profile=` argument — either a partial dict (merged on top) or a `Callable[[ModelProfile], ModelProfile]` (full control: receives the resolved default, returns the final profile).
+
+##### Resolved profiles now carry cross-class fields
+
+In v1, `ModelProfile.update()` silently filtered out fields not declared on the target class. In v2, dict-spread preserves every key.
+
+This means e.g. a Bedrock-hosted Anthropic model's resolved profile now carries the upstream `anthropic_*` fields alongside the `bedrock_*` fields, where v1 dropped them. No in-tree model class reads cross-class fields, so behavior is unchanged in the standard providers; but custom model classes that do `profile.get('anthropic_supports_adaptive_thinking', False)` on a non-Anthropic route will now see the value the upstream Anthropic profile set, where v1 always returned the default.
+
+See the [Model Profile guide](models/openai.md#model-profile) for how to configure a profile, and [PR #5481](https://github.com/pydantic/pydantic-ai/pull/5481) for the full `ModelProfile` redesign.
+
+##### Parallel tool-call execution runs in emission order
+
+The default [`end_strategy`][pydantic_ai.agent.EndStrategy] changed from `'early'` to `'graceful'`. This only affects responses where a model calls function tools in the *same* response as an [output tool](output.md#tool-output) (the call that ends the run). When that output tool **succeeds**, the function tools requested alongside it now **run** by default instead of being skipped, so their side effects happen and their results reach the model if the run continues; and a function tool's [`ModelRetry`][pydantic_ai.exceptions.ModelRetry] now suppresses the output result so the model can correct itself on the next round. The case where *every* output tool fails is unchanged: function tools run and the run continues either way. Most agents don't need any change. If you relied on the run ending the instant an output tool succeeds — skipping any function tools requested in the same response — set `end_strategy='early'` explicitly.
+
+The [`sequential=True`](tools-advanced.md#parallel-tool-calls-concurrency) flag on a tool is now a per-tool **barrier** rather than a batch-wide serial switch: a sequential tool runs alone, but other tools in the same response still run in parallel around it. The barrier now also applies to output tools via [`ToolOutput(sequential=True)`][pydantic_ai.output.ToolOutput], not just function tools. To run *all* of a run's tools serially, wrap the run in [`agent.parallel_tool_call_execution_mode('sequential')`][pydantic_ai.agent.AbstractAgent.parallel_tool_call_execution_mode] or set `parallel_tool_calls=False` on the [model settings][pydantic_ai.settings.ModelSettings].
+
+See [Parallel Output Tool Calls](output.md#parallel-output-tool-calls) for the full behavior of all three strategies, and [#5339](https://github.com/pydantic/pydantic-ai/pull/5339).
+
+##### Slimmer default `pydantic-ai` extras
+
+A bare `uv add pydantic-ai` / `pip install pydantic-ai` now installs `pydantic-ai-slim[openai,anthropic,google,cli,mcp,evals,web,retries,logfire]` — frontier providers plus minimal integrations. Providers and integrations that were previously bundled are no longer installed by default; add the ones you use explicitly, e.g. `uv add 'pydantic-ai[bedrock,groq]'`: `bedrock`, `groq`, `mistral`, `cohere`, `xai`, `huggingface`, `temporal`, `ag-ui`, `ui`, and `spec`. See the [installation guide](install.md) for the full list of extras.
+
+Some `pydantic-ai-slim` extras were also removed outright (not just dropped from the default bundle): the `outlines-*` extras (the Outlines integration is removed), `vertexai` (Vertex AI is now served by the `google` extra), `fastmcp` (the FastMCP back-compat shim is removed), and `a2a` (A2A now lives in the upstream `fasta2a` package). See [#5467](https://github.com/pydantic/pydantic-ai/pull/5467).
+
+##### Instrumentation defaults to version 5 with aggregated usage attributes
+
+The default [instrumentation format](logfire.md#configuring-data-format) is now version 5 (versions 2–4 still work but emit a deprecation warning; version 1 and its `event_mode=`/`logger_provider=` arguments are removed). In version 5, deferred tool calls (`CallDeferred`/`ApprovalRequired`) are no longer recorded as span errors.
+
+Separately, [`InstrumentationSettings`][pydantic_ai.models.instrumented.InstrumentationSettings]'s [`use_aggregated_usage_attribute_names`](logfire.md#aggregated-usage-attribute-names) now defaults to `True`: agent run spans report token usage under `gen_ai.aggregated_usage.*` while model request spans keep `gen_ai.usage.*`, which avoids double-counting in backends that sum parent and child usage. Dashboards and alerts that read token usage from run spans must be updated, or set `use_aggregated_usage_attribute_names=False` to keep the V1 attribute names.
+
+See [#5523](https://github.com/pydantic/pydantic-ai/pull/5523).
+
+#### Changes covered by deprecation warnings
+
+These changes were announced in the latest V1 releases via deprecation warnings that name the replacement API. If you upgraded to the latest V1 and resolved every warning, you've already made them; they're listed here with full before → after for reference.
+
+**Behavior changes that flip silently if the V1 deprecation warning was not addressed** — even though these were announced, an unaddressed warning means the behavior changes without raising an error, so confirm you've handled them:
+
+- The bare `openai:` model prefix now uses the OpenAI Responses API ([`OpenAIResponsesModel`][pydantic_ai.models.openai.OpenAIResponsesModel]) instead of the Chat Completions API ([`OpenAIChatModel`][pydantic_ai.models.openai.OpenAIChatModel]). Use `openai-chat:` to keep Chat Completions, or `openai-responses:` to opt into the new default explicitly. Announced via [#5334](https://github.com/pydantic/pydantic-ai/pull/5334); flipped in [#5469](https://github.com/pydantic/pydantic-ai/pull/5469).
+- Provider-adaptive `WebSearch` and `WebFetch` capabilities are now native-only and raise on models that don't support them, and `MCP(url=...)` runs the server locally by default. Restore the V1 fallbacks with `WebSearch(local='duckduckgo')`, `WebFetch(local=True)`, and `MCP(url=..., native=True)`. Announced via [#5331](https://github.com/pydantic/pydantic-ai/pull/5331); changed in [#5333](https://github.com/pydantic/pydantic-ai/pull/5333).
+
+**API removals and renames:**
+
+- `pydantic_ai.providers.grok.GrokProvider` and `pydantic_ai.providers.grok.GrokModelName` are removed; use `pydantic_ai.providers.xai.XaiProvider` with `pydantic_ai.models.xai.XaiModel` (and `pydantic_ai.models.xai.XaiModelName`). The `grok:` model prefix is removed; use `xai:`. See [#5460](https://github.com/pydantic/pydantic-ai/pull/5460).
+- `GoogleGLAProvider`, `GoogleVertexProvider`, and `GeminiModel` (the whole `pydantic_ai.models.gemini` module) are removed; use `pydantic_ai.providers.google.GoogleProvider` (Gemini API) or `pydantic_ai.providers.google_cloud.GoogleCloudProvider` (Vertex) with `pydantic_ai.models.google.GoogleModel`. Provider prefixes: `google-gla:`→`google:`, `google-vertex:`→`google-cloud:`, `vertexai:`→`google-cloud:`, and `gateway/gemini:`/`gateway/google-vertex:`→`gateway/google-cloud:`. `GoogleProvider(vertexai=, location=, project=, credentials=)` → `GoogleCloudProvider(...)`. `GoogleModelSettings` keys `google_vertex_service_tier`/`google_service_tier` → `google_cloud_service_tier`. Announced via [#5336](https://github.com/pydantic/pydantic-ai/pull/5336) and [#5543](https://github.com/pydantic/pydantic-ai/pull/5543); removed in [#5479](https://github.com/pydantic/pydantic-ai/pull/5479).
+- `OpenAIModel` → `OpenAIChatModel`, `OpenAIModelSettings` → `OpenAIChatModelSettings`; the `OpenAIChatModel(system_prompt_role=...)` kwarg → `OpenAIModelProfile(openai_system_prompt_role=...)`; `OpenAICompaction(instructions=...)` removed; `OpenAIModelProfile.openai_supports_sampling_settings` → `openai_unsupported_model_settings`. See [#5468](https://github.com/pydantic/pydantic-ai/pull/5468).
+- Built-in tools are renamed to "native" tools: `pydantic_ai.builtin_tools` → `pydantic_ai.native_tools`; `BuiltinToolCallPart`/`BuiltinToolReturnPart`/`AgentBuiltinTool` → `NativeToolCallPart`/`NativeToolReturnPart`/`AgentNativeTool`; `Agent(builtin_tools=[...])` → `capabilities=[NativeTool(...)]`; `builtin=` → `native=`; `OpenAIModelProfile.openai_builtin_tools` → `openai_native_tools`. The serialized `part_kind` wire values are unchanged, so message history still deserializes. Announced via [#5338](https://github.com/pydantic/pydantic-ai/pull/5338); removed in [#5396](https://github.com/pydantic/pydantic-ai/pull/5396).
+- MCP: `MCPServerStdio`/`MCPServerSSE`/`MCPServerStreamableHTTP`/`MCPServerHTTP`, `FastMCPToolset`, `load_mcp_servers`, `Agent.run_mcp_servers()`, and `Agent.set_mcp_sampling_model()` are removed; use `pydantic_ai.mcp.MCPToolset`, `pydantic_ai.mcp.load_mcp_toolsets`, `async with agent:`, and `MCPToolset(sampling_model=...)`. Note that the new `MCPToolset` defaults differ (e.g. `max_retries`, `read_timeout`, `init_timeout`, `elicitation_handler`). Announced via [#5325](https://github.com/pydantic/pydantic-ai/pull/5325); removed in [#5337](https://github.com/pydantic/pydantic-ai/pull/5337).
+- `Agent(instrument=...)`, `Agent.from_spec(instrument=...)`, `Agent.from_file(instrument=...)`, and `AgentSpec.instrument` are removed; use `capabilities=[Instrumentation(...)]`. (The `Agent.instrument` property, `Agent.instrument_all()`, and `InstrumentedModel` are unchanged.) See [#5434](https://github.com/pydantic/pydantic-ai/pull/5434).
+- `Agent(event_stream_handler=...)` → `capabilities=[ProcessEventStream(...)]`; `Agent(prepare_tools=...)` → `capabilities=[PrepareTools(...)]`. The `event_stream_handler=` argument on `run()`/`run_sync()`/`run_stream()`/`iter()` is unchanged. Announced via [#5335](https://github.com/pydantic/pydantic-ai/pull/5335); removed in [#5475](https://github.com/pydantic/pydantic-ai/pull/5475).
+- `Agent(history_processors=...)` → `capabilities=[ProcessHistory(...)]`. See [#5425](https://github.com/pydantic/pydantic-ai/pull/5425).
+- `Agent(mcp_servers=[...])` → `Agent(toolsets=[...])`; `Agent.sequential_tool_calls()` → `agent.parallel_tool_call_execution_mode('sequential')`. See [#5466](https://github.com/pydantic/pydantic-ai/pull/5466).
+- `Agent.to_a2a()` and the bundled `fasta2a` integration (and the `[a2a]` extra) are removed; install `fasta2a[pydantic-ai]>=0.6.1` and use `from fasta2a.pydantic_ai import agent_to_a2a`. Announced via [#5426](https://github.com/pydantic/pydantic-ai/pull/5426); removed in [#5502](https://github.com/pydantic/pydantic-ai/pull/5502).
+- `Agent.to_ag_ui()`, `AGUIApp`, and the `pydantic_ai.ag_ui` shim are removed; use `pydantic_ai.ui.ag_ui.AGUIAdapter`. `pydantic_ai.models.cached_async_http_client` is removed; use `pydantic_ai.models.create_async_http_client()` or your own `httpx.AsyncClient`. Announced via [#5345](https://github.com/pydantic/pydantic-ai/pull/5345); removed in [#5464](https://github.com/pydantic/pydantic-ai/pull/5464).
+- `pydantic_ai.ext.aci` (`tool_from_aci`, `ACIToolset`) is removed with no upstream replacement; wrap ACI tools with `Tool.from_schema`. Announced via [#5510](https://github.com/pydantic/pydantic-ai/pull/5510); removed in [#5467](https://github.com/pydantic/pydantic-ai/pull/5467).
+- `pydantic_ai.output.DeferredToolCalls` → `DeferredToolRequests`; `pydantic_ai.toolsets.external.DeferredToolset` → `ExternalToolset`. See [#5459](https://github.com/pydantic/pydantic-ai/pull/5459).
+- `FunctionToolset.tool()` now raises if the decorated callable's first parameter is not a `RunContext`; use `FunctionToolset.tool_plain()` for context-free tools. See [#5462](https://github.com/pydantic/pydantic-ai/pull/5462).
+- Usage/token renames: `request_tokens` → `input_tokens`, `response_tokens` → `output_tokens`, `Usage` → `RunUsage`, `UsageLimits(request_tokens_limit=)` → `input_tokens_limit=`, `UsageLimits(response_tokens_limit=)` → `output_tokens_limit=`. Response field renames: `ModelResponse.vendor_details` → `provider_details`, `vendor_id`/`provider_request_id` → `provider_response_id`. Removed event-class shims `BuiltinToolCallEvent`/`BuiltinToolResultEvent`, and `FunctionToolCallEvent.call_id` → `.tool_call_id`. Message history serialized with the old field names still deserializes via retained validation aliases. See [#5476](https://github.com/pydantic/pydantic-ai/pull/5476).
+- Output tool calls now emit dedicated `OutputToolCallEvent`/`OutputToolResultEvent` rather than `FunctionToolCallEvent`/`FunctionToolResultEvent`; `FunctionToolResultEvent(result=...)`/`.result` → `(part=...)`/`.part`. See [#5332](https://github.com/pydantic/pydantic-ai/pull/5332).
+- `StreamedRunResult.stream` → `stream_output`, `StreamedRunResult.stream_structured` → `stream_response`, `StreamedRunResult.validate_structured_output` → `validate_response_output`; the plural `stream_responses()` → singular `stream_response()` (which yields a bare `ModelResponse`; read the old `is_last` flag as `response.state != 'incomplete'`). Announced via [#5296](https://github.com/pydantic/pydantic-ai/pull/5296); removed in [#5463](https://github.com/pydantic/pydantic-ai/pull/5463).
+- `result.usage()` → `result.usage`, `result.timestamp()` → `result.timestamp`, and `stream.get()` → `stream.response` (method-style accessors become properties). See [#5263](https://github.com/pydantic/pydantic-ai/pull/5263).
+- `StreamedResponse.usage()` → `StreamedResponse.usage`: the model-adapter streaming base class (`pydantic_ai.models.StreamedResponse`) now exposes `usage` as a property rather than a method. Relevant if you've subclassed `Model` and call `.usage()` on a streamed response. See [#5546](https://github.com/pydantic/pydantic-ai/pull/5546).
+- `pydantic_graph.beta` imports move to the top-level `pydantic_graph` (e.g. `from pydantic_graph import GraphBuilder`). Announced via [#5306](https://github.com/pydantic/pydantic-ai/pull/5306); removed in [#5470](https://github.com/pydantic/pydantic-ai/pull/5470).
+- Instrumentation format `version=1` and its version-1-only `InstrumentationSettings(event_mode=...)` and `InstrumentationSettings(logger_provider=...)` arguments are removed (deprecated in V1); `version=2`/`3`/`4` still work but now emit a deprecation warning. The default is `version=5` — see [Instrumentation defaults](#instrumentation-defaults-to-version-5-with-aggregated-usage-attributes) above for the default-behavior changes that ship with it. See [#5523](https://github.com/pydantic/pydantic-ai/pull/5523).
+- The Outlines integration (`pydantic_ai.models.outlines.OutlinesModel`, `pydantic_ai.providers.outlines.OutlinesProvider`, and the `outlines-*` extras) is removed. If you'd like to keep using Outlines with Pydantic AI, please file an issue at [dottxt-ai/outlines](https://github.com/dottxt-ai/outlines/issues). See [#5444](https://github.com/pydantic/pydantic-ai/pull/5444).
+- `pydantic_ai.native_tools.UrlContextTool` is removed; use `pydantic_ai.native_tools.WebFetchTool` instead. See [#5458](https://github.com/pydantic/pydantic-ai/pull/5458).
+- Iterating [`Agent.run_stream_events()`][pydantic_ai.agent.AbstractAgent.run_stream_events] directly is no longer supported; it is now an async context manager only: `async with agent.run_stream_events(...) as events: async for event in events: ...`. See [#5440](https://github.com/pydantic/pydantic-ai/pull/5440).
+- The bare (provider-prefix-less) model-name fallback is removed: `Agent('gpt-5')` now raises a `UserError` instead of inferring the provider; pass a provider-prefixed model name like `Agent('openai:gpt-5')`. (V1 emitted a deprecation warning for prefix-less legacy model names.) See [#5464](https://github.com/pydantic/pydantic-ai/pull/5464).
+- Pydantic Evals: `EvaluationResult` and `EvaluatorFailure` are now keyword-only; `Dataset.evaluate()`/`evaluate_sync()` make `name`/`max_concurrency`/`progress`/`retry_task`/`retry_evaluators` keyword-only; `Dataset(name=...)` is now required; the `Evaluator.name` classmethod → `Evaluator.get_serialization_name()`. Announced via [#5547](https://github.com/pydantic/pydantic-ai/pull/5547); changed in [#5548](https://github.com/pydantic/pydantic-ai/pull/5548).
+- Pydantic Evals: custom `Evaluator`s that advertised a default name or version by setting an `evaluation_name` / `evaluator_version` class attribute should override `Evaluator.get_default_evaluation_name()` / `Evaluator.get_evaluator_version()` instead; the attribute fallback is removed. Announced via [#5554](https://github.com/pydantic/pydantic-ai/pull/5554); removed in [#5556](https://github.com/pydantic/pydantic-ai/pull/5556).
+
+### v1.0.1 (2025-09-05)
+
+The following breaking change was accidentally left out of v1.0.0:
+
+- See [#2808](https://github.com/pydantic/pydantic-ai/pull/2808) - Remove `Python` evaluator from `pydantic_evals` for security reasons
+
+### v1.0.0 (2025-09-04)
+
+- See [#2725](https://github.com/pydantic/pydantic-ai/pull/2725) - Drop support for Python 3.9
+- See [#2738](https://github.com/pydantic/pydantic-ai/pull/2738) - Make many dataclasses require keyword arguments
+- See [#2715](https://github.com/pydantic/pydantic-ai/pull/2715) - Remove `cases` and `averages` attributes from `pydantic_evals` spans
+- See [#2798](https://github.com/pydantic/pydantic-ai/pull/2798) - Change `ModelRequest.parts` and `ModelResponse.parts` types from `list` to `Sequence`
+- See [#2726](https://github.com/pydantic/pydantic-ai/pull/2726) - Default `InstrumentationSettings` version to 2
+- See [#2717](https://github.com/pydantic/pydantic-ai/pull/2717) - Remove errors when passing `AsyncRetrying` or `Retrying` object to `AsyncTenacityTransport` or `TenacityTransport` instead of `RetryConfig`
+
+### v0.x.x
+
+Before V1, minor versions were used to introduce breaking changes:
+
+**v0.8.0 (2025-08-26)**
+
+See [#2689](https://github.com/pydantic/pydantic-ai/pull/2689) - `AgentStreamEvent` was expanded to be a union of `ModelResponseStreamEvent` and `HandleResponseEvent`, simplifying the `event_stream_handler` function signature. Existing code accepting `AgentStreamEvent | HandleResponseEvent` will continue to work.
+
+**v0.7.6 (2025-08-26)**
+
+The following breaking change was inadvertently released in a patch version rather than a minor version:
+
+See [#2670](https://github.com/pydantic/pydantic-ai/pull/2670) - `TenacityTransport` and `AsyncTenacityTransport` now require the use of `pydantic_ai.retries.RetryConfig` (which is just a `TypedDict` containing the kwargs to `tenacity.retry`) instead of `tenacity.Retrying` or `tenacity.AsyncRetrying`.
+
+**v0.7.0 (2025-08-12)**
+
+See [#2458](https://github.com/pydantic/pydantic-ai/pull/2458) - `pydantic_ai.models.StreamedResponse` now yields a `FinalResultEvent` along with the existing `PartStartEvent` and `PartDeltaEvent`. If you're using `pydantic_ai.direct.model_request_stream` or `pydantic_ai.direct.model_request_stream_sync`, you may need to update your code to account for this.
+
+See [#2458](https://github.com/pydantic/pydantic-ai/pull/2458) - `pydantic_ai.models.Model.request_stream` now receives a `run_context` argument. If you've implemented a custom `Model` subclass, you will need to account for this.
+
+See [#2458](https://github.com/pydantic/pydantic-ai/pull/2458) - `pydantic_ai.models.StreamedResponse` now requires a `model_request_parameters` field and constructor argument. If you've implemented a custom `Model` subclass and implemented `request_stream`, you will need to account for this.
+
+**v0.6.0 (2025-08-06)**
+
+This release was meant to clean some old deprecated code, so we can get a step closer to V1.
+
+See [#2440](https://github.com/pydantic/pydantic-ai/pull/2440) - The `next` method was removed from the `Graph` class. Use `async with graph.iter(...) as run:  run.next()` instead.
+
+See [#2441](https://github.com/pydantic/pydantic-ai/pull/2441) - The `result_type`, `result_tool_name` and `result_tool_description` arguments were removed from the `Agent` class. Use `output_type` instead.
+
+See [#2441](https://github.com/pydantic/pydantic-ai/pull/2441) - The `result_retries` argument was also removed from the `Agent` class. Use `output_retries` instead.
+
+See [#2443](https://github.com/pydantic/pydantic-ai/pull/2443) - The `data` property was removed from the `FinalResult` class. Use `output` instead.
+
+See [#2445](https://github.com/pydantic/pydantic-ai/pull/2445) - The `get_data` and `validate_structured_result` methods were removed from the
+`StreamedRunResult` class. Use `get_output` and `validate_response_output` instead.
+
+See [#2446](https://github.com/pydantic/pydantic-ai/pull/2446) - The `format_as_xml` function was moved to the `pydantic_ai.format_as_xml` module.
+Import it via `from pydantic_ai import format_as_xml` instead.
+
+See [#2451](https://github.com/pydantic/pydantic-ai/pull/2451) - Removed deprecated `Agent.result_validator` method, `Agent.last_run_messages` property, `AgentRunResult.data` property, and `result_tool_return_content` parameters from result classes.
+
+**v0.5.0 (2025-08-04)**
+
+See [#2388](https://github.com/pydantic/pydantic-ai/pull/2388) - The `source` field of an `EvaluationResult` is now of type `EvaluatorSpec` rather than the actual source `Evaluator` instance, to help with serialization/deserialization.
+
+See [#2163](https://github.com/pydantic/pydantic-ai/pull/2163) - The `EvaluationReport.print` and `EvaluationReport.console_table` methods now require most arguments be passed by keyword.
+
+**v0.4.0 (2025-07-08)**
+
+See [#1799](https://github.com/pydantic/pydantic-ai/pull/1799) - Pydantic Evals `EvaluationReport` and `ReportCase` are now generic dataclasses instead of Pydantic models. If you were serializing them using `model_dump()`, you will now need to use the `EvaluationReportAdapter` and `ReportCaseAdapter` type adapters instead.
+
+See [#1507](https://github.com/pydantic/pydantic-ai/pull/1507) - The `ToolDefinition` `description` argument is now optional and the order of positional arguments has changed from `name, description, parameters_json_schema, ...` to `name, parameters_json_schema, description, ...` to account for this.
+
+**v0.3.0 (2025-06-18)**
+
+See [#1142](https://github.com/pydantic/pydantic-ai/pull/1142) — Adds support for thinking parts.
+
+We now convert the thinking blocks (`"<think>..."</think>"`) in provider specific text parts to
+Pydantic AI `ThinkingPart`s. Also, as part of this release, we made the choice to not send back the
+`ThinkingPart`s to the provider - the idea is to save costs on behalf of the user. In the future, we
+intend to add a setting to customize this behavior.
+
+**v0.2.0 (2025-05-12)**
+
+See [#1647](https://github.com/pydantic/pydantic-ai/pull/1647) — usage makes sense as part of `ModelResponse`, and could be really useful in "messages" (really a sequence of requests and response). In this PR:
+
+- Adds `usage` to `ModelResponse` (field has a default factory of `Usage()` so it'll work to load data that doesn't have usage)
+- changes the return type of `Model.request` to just `ModelResponse` instead of `tuple[ModelResponse, Usage]`
+
+**v0.1.0 (2025-04-15)**
+
+See [#1248](https://github.com/pydantic/pydantic-ai/pull/1248) — the attribute/parameter name `result` was renamed to `output` in many places. Hopefully all changes keep a deprecated attribute or parameter with the old name, so you should get many deprecation warnings.
+
+See [#1484](https://github.com/pydantic/pydantic-ai/pull/1484) — `format_as_xml` was moved and made available to import from the package root, e.g. `from pydantic_ai import format_as_xml`.
+
+## Full Changelog
+
+<div id="display-changelog">
+  For the full changelog, see <a href="https://github.com/pydantic/pydantic-ai/releases">GitHub Releases</a>.
+</div>
+
+<script>
+  fetch('/changelog.html').then(r => {
+    if (r.ok) {
+      r.text().then(t => {
+        document.getElementById('display-changelog').innerHTML = t;
+      });
+    }
+  });
+</script>

docs/cli.md

@@ -0,0 +1,202 @@
+# Command Line Interface (CLI)
+
+**Pydantic AI** comes with a CLI, `clai` (pronounced "clay"). You can use it to chat with various LLMs and quickly get answers, right from the command line, or spin up a uvicorn server to chat with your Pydantic AI agents from your browser.
+
+## Installation
+
+You can run the `clai` using [`uvx`](https://docs.astral.sh/uv/guides/tools/):
+
+```bash
+uvx clai
+```
+
+Or install `clai` globally [with `uv`](https://docs.astral.sh/uv/guides/tools/#installing-tools):
+
+```bash
+uv tool install clai
+...
+clai
+```
+
+Or with `pip`:
+
+```bash
+pip install clai
+...
+clai
+```
+
+## CLI Usage
+
+<!-- clai/README.md links here for full docs -->
+
+You'll need to set an environment variable depending on the provider you intend to use.
+
+E.g. if you're using OpenAI, set the `OPENAI_API_KEY` environment variable:
+
+```bash
+export OPENAI_API_KEY='your-api-key-here'
+```
+
+Then running `clai` will start an interactive session where you can chat with the AI model. Special commands available in interactive mode:
+
+- `/exit`: Exit the session
+- `/markdown`: Show the last response in markdown format
+- `/multiline`: Toggle multiline input mode (use Ctrl+D to submit)
+- `/cp`: Copy the last response to clipboard
+- `/usage`: Show cumulative token usage for the session (turns, input, output, requests, tool calls); add `--json` for a single-line JSON object
+
+### CLI Options
+
+| Option | Description |
+|--------|-------------|
+| `prompt` | AI prompt for one-shot mode (positional). If omitted, starts interactive mode. |
+| `-m`, `--model` | Model to use in `provider:model` format (e.g., `openai:gpt-5.2`) |
+| `-a`, `--agent` | Custom agent in `module:variable` format |
+| `-t`, `--code-theme` | Syntax highlighting theme (`dark`, `light`, or [pygments theme](https://pygments.org/styles/)) |
+| `--no-stream` | Disable streaming from the model |
+| `-l`, `--list-models` | List all available models and exit |
+| `--version` | Show version and exit |
+
+### Choose a model
+
+You can specify which model to use with the `--model` flag:
+
+```bash
+clai --model anthropic:claude-sonnet-4-6
+```
+
+(a full list of models available can be printed with `clai --list-models`)
+
+### Custom Agents
+
+You can specify a custom agent using the `--agent` flag with a module path and variable name:
+
+```python {title="custom_agent.py" test="skip"}
+from pydantic_ai import Agent
+
+agent = Agent('openai:gpt-5.2', instructions='You always respond in Italian.')
+```
+
+Then run:
+
+```bash
+clai --agent custom_agent:agent "What's the weather today?"
+```
+
+The format must be `module:variable` where:
+
+- `module` is the importable Python module path
+- `variable` is the name of the Agent instance in that module
+
+Additionally, you can directly launch CLI mode from an `Agent` instance using `Agent.to_cli_sync()`:
+
+```python {title="agent_to_cli_sync.py" test="skip" hl_lines=4}
+from pydantic_ai import Agent
+
+agent = Agent('openai:gpt-5.2', instructions='You always respond in Italian.')
+agent.to_cli_sync()
+```
+
+You can also use the async interface with `Agent.to_cli()`:
+
+```python {title="agent_to_cli.py" test="skip" hl_lines=6}
+from pydantic_ai import Agent
+
+agent = Agent('openai:gpt-5.2', instructions='You always respond in Italian.')
+
+async def main():
+    await agent.to_cli()
+```
+
+_(You'll need to add `asyncio.run(main())` to run `main`)_
+
+### Message History
+
+Both `Agent.to_cli()` and `Agent.to_cli_sync()` support a `message_history` parameter, allowing you to continue an existing conversation or provide conversation context:
+
+```python {title="agent_with_history.py" test="skip"}
+from pydantic_ai import (
+    Agent,
+    ModelMessage,
+    ModelRequest,
+    ModelResponse,
+    TextPart,
+    UserPromptPart,
+)
+
+agent = Agent('openai:gpt-5.2')
+
+# Create some conversation history
+message_history: list[ModelMessage] = [
+    ModelRequest([UserPromptPart(content='What is 2+2?')]),
+    ModelResponse([TextPart(content='2+2 equals 4.')])
+]
+
+# Start CLI with existing conversation context
+agent.to_cli_sync(message_history=message_history)
+```
+
+The CLI will start with the provided conversation history, allowing the agent to refer back to previous exchanges and maintain context throughout the session.
+
+## Web Chat UI
+
+Launch a web-based chat interface by running:
+
+```bash
+clai web -m openai:gpt-5.2
+```
+
+This will start a web server (default: http://127.0.0.1:7932) with a chat interface.
+
+You can also serve an existing agent. For example, if you have an agent defined in `my_agent.py`:
+
+```python
+from pydantic_ai import Agent
+
+my_agent = Agent('openai:gpt-5.2', instructions='You are a helpful assistant.')
+```
+
+Launch the web UI:
+
+```bash
+# With a custom agent
+clai web --agent my_module:my_agent
+
+# With specific models (first is default when no --agent)
+clai web -m openai:gpt-5.2 -m anthropic:claude-sonnet-4-6
+
+# With native tools
+clai web -m openai:gpt-5.2 -t web_search -t code_execution
+
+# Generic agent with system instructions
+clai web -m openai:gpt-5.2 -i 'You are a helpful coding assistant'
+
+# Custom agent with extra instructions for each run
+clai web --agent my_module:my_agent -i 'Always respond in Spanish'
+```
+
+!!! note "Memory Tool"
+    The [`memory`](native-tools.md#memory-tool) native tool cannot be enabled via `-t memory`. If your agent needs memory, configure the [`MemoryTool`][pydantic_ai.native_tools.MemoryTool] directly on the agent and provide it via `--agent`.
+
+### Web UI Options
+
+| Option | Description |
+|--------|-------------|
+| `--agent`, `-a` | Agent to serve in [`module:variable` format](#custom-agents) |
+| `--model`, `-m` | Models to list as options in the UI (repeatable) |
+| `--tool`, `-t` | [Native tool](native-tools.md)s to list as options in the UI (repeatable). See [available tools](web.md#native-tool-support). |
+| `--instructions`, `-i` | System instructions. When `--agent` is specified, these are additional to the agent's existing instructions. |
+| `--host` | Host to bind server (default: 127.0.0.1) |
+| `--port` | Port to bind server (default: 7932) |
+| `--html-source` | URL or file path for the chat UI HTML. |
+
+When using `--agent`, the agent's configured model becomes the default. CLI models (`-m`) are additional options. Without `--agent`, the first `-m` model is the default.
+
+The web chat UI can also be launched programmatically using [`Agent.to_web()`][pydantic_ai.agent.Agent.to_web], see the [Web UI documentation](web.md).
+
+Run the `web` command with `--help` to see all available options:
+
+```bash
+clai web --help
+```

docs/coding-agent-skills.md

@@ -0,0 +1,53 @@
+# Coding Agent Skills
+
+If you're building Pydantic AI applications with a coding agent, you can install the Pydantic AI skill from the [`pydantic/skills`](https://github.com/pydantic/skills) repository to give your agent up-to-date framework knowledge.
+
+[Agent skills](https://agentskills.io) are packages of instructions and reference material that coding agents load on demand. With the skill installed, coding agents have access to Pydantic AI patterns, architecture guidance, and common task references covering [tools](tools.md), [capabilities](capabilities.md), [structured output](output.md), [streaming](agent.md#streaming-events-and-final-output), [testing](testing.md), [multi-agent delegation](multi-agent-applications.md), [hooks](hooks.md), and [agent specs](agent-spec.md).
+
+!!! note
+    If you want to build agent skills for your Pydantic AI agent, see the [Agent Skills](capabilities.md#agent-skills) entry in the Third-party capabilities section on the Capabilities page.
+
+## Installation
+
+### Claude Code
+
+Install the [official Pydantic AI plugin](https://claude.com/plugins/pydantic-ai) from the Anthropic marketplace, which is available by default:
+
+```bash
+claude plugin install pydantic-ai@claude-plugins-official
+```
+
+As an alternative, you can install from the [`pydantic/skills`](https://github.com/pydantic/skills) marketplace, which bundles the Pydantic AI skill alongside other Pydantic-maintained skills:
+
+```bash
+claude plugin marketplace add pydantic/skills
+claude plugin install ai@pydantic-skills
+```
+
+### Cross-Agent (agentskills.io)
+
+Install the Pydantic AI skill using the [skills CLI](https://github.com/vercel-labs/skills):
+
+```bash
+npx skills add pydantic/skills
+```
+
+This works with 30+ agents via the [agentskills.io](https://agentskills.io) standard, including Claude Code, Codex, Cursor, and Gemini CLI.
+
+### Library Skills
+
+Pydantic AI also ships its skill bundled with the package, so you can install it directly from your project's dependencies via [library-skills.io](https://library-skills.io):
+
+```bash
+uvx library-skills --all
+```
+
+The `--all` flag is required because the skill is bundled in `pydantic-ai-slim`, which is a transitive dependency of the `pydantic-ai` meta-package. Without it, `library-skills` only scans direct dependencies and won't discover the skill.
+
+Add `--claude` to also install into `.claude/skills/` alongside the default `.agents/skills/` directory, since Claude Code doesn't read from `.agents/`.
+
+## See Also
+
+- [`pydantic/skills`](https://github.com/pydantic/skills): source repository
+- [agentskills.io](https://agentskills.io): the open standard for agent skills
+- [library-skills.io](https://library-skills.io): install agent skills bundled with your project's dependencies

docs/common-tools.md

@@ -0,0 +1,292 @@
+# Common Tools
+
+Pydantic AI ships with native tools that can be used to enhance your agent's capabilities.
+
+## DuckDuckGo Search Tool
+
+The DuckDuckGo search tool allows you to search the web for information. It is built on top of the
+[DuckDuckGo API](https://github.com/deedy5/ddgs).
+
+### Installation
+
+To use [`duckduckgo_search_tool`][pydantic_ai.common_tools.duckduckgo.duckduckgo_search_tool], you need to install
+[`pydantic-ai-slim`](install.md#slim-install) with the `duckduckgo` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[duckduckgo]"
+```
+
+### Usage
+
+Here's an example of how you can use the DuckDuckGo search tool with an agent:
+
+```py {title="duckduckgo_search.py" test="skip"}
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.duckduckgo import duckduckgo_search_tool
+
+agent = Agent(
+    'openai:gpt-5.2',
+    tools=[duckduckgo_search_tool()],
+    instructions='Search DuckDuckGo for the given query and return the results.',
+)
+
+result = agent.run_sync(
+    'Can you list the top five highest-grossing animated films of 2025?'
+)
+print(result.output)
+"""
+I looked into several sources on animated box‐office performance in 2025, and while detailed
+rankings can shift as more money is tallied, multiple independent reports have already
+highlighted a couple of record‐breaking shows. For example:
+
+• Ne Zha 2 – News outlets (Variety, Wikipedia's "List of animated feature films of 2025", and others)
+    have reported that this Chinese title not only became the highest‑grossing animated film of 2025
+    but also broke records as the highest‑grossing non‑English animated film ever. One article noted
+    its run exceeded US$1.7 billion.
+• Inside Out 2 – According to data shared on Statista and in industry news, this Pixar sequel has been
+    on pace to set new records (with some sources even noting it as the highest‑grossing animated film
+    ever, as of January 2025).
+
+Beyond those two, some entertainment trade sites (for example, a Just Jared article titled
+"Top 10 Highest-Earning Animated Films at the Box Office Revealed") have begun listing a broader
+top‑10. Although full consolidated figures can sometimes differ by source and are updated daily during
+a box‑office run, many of the industry trackers have begun to single out five films as the biggest
+earners so far in 2025.
+
+Unfortunately, although multiple articles discuss the "top animated films" of 2025, there isn't yet a
+single, universally accepted list with final numbers that names the complete top five. (Box‑office
+rankings, especially mid‑year, can be fluid as films continue to add to their totals.)
+
+Based on what several sources note so far, the two undisputed leaders are:
+1. Ne Zha 2
+2. Inside Out 2
+
+The remaining top spots (3–5) are reported by some outlets in their "Top‑10 Animated Films"
+lists for 2025 but the titles and order can vary depending on the source and the exact cut‑off
+date of the data. For the most up‑to‑date and detailed ranking (including the 3rd, 4th, and 5th
+highest‑grossing films), I recommend checking resources like:
+• Wikipedia's "List of animated feature films of 2025" page
+• Box‑office tracking sites (such as Box Office Mojo or The Numbers)
+• Trade articles like the one on Just Jared
+
+To summarize with what is clear from the current reporting:
+1. Ne Zha 2
+2. Inside Out 2
+3–5. Other animated films (yet to be definitively finalized across all reporting outlets)
+
+If you're looking for a final, consensus list of the top five, it may be best to wait until
+the 2025 year‑end box‑office tallies are in or to consult a regularly updated entertainment industry source.
+
+Would you like help finding a current source or additional details on where to look for the complete updated list?
+"""
+```
+
+## Web Fetch Tool
+
+The web fetch tool allows your agent to fetch the content of web pages and convert them to markdown.
+It uses [SSRF protection](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery) to prevent server-side request forgery attacks.
+
+### Installation
+
+To use [`web_fetch_tool`][pydantic_ai.common_tools.web_fetch.web_fetch_tool], you need to install
+[`pydantic-ai-slim`](install.md#slim-install) with the `web-fetch` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[web-fetch]"
+```
+
+### Usage
+
+Here's an example of how you can use the web fetch tool with an agent:
+
+```py {title="web_fetch.py" test="skip"}
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.web_fetch import web_fetch_tool
+
+agent = Agent(
+    'openai:gpt-5.2',
+    tools=[web_fetch_tool()],
+    instructions='Fetch web pages and summarize their content.',
+)
+
+result = agent.run_sync('What is on https://ai.pydantic.dev?')
+print(result.output)
+```
+
+!!! tip "Automatic fallback via WebFetch capability"
+    You don't need to use [`web_fetch_tool`][pydantic_ai.common_tools.web_fetch.web_fetch_tool] directly — the
+    [`WebFetch`][pydantic_ai.capabilities.WebFetch] capability automatically uses it
+    as a local fallback when the model doesn't support native URL fetching.
+
+## Tavily Search Tool
+
+!!! info
+    Tavily is a paid service, but they have free credits to explore their product.
+
+    You need to [sign up for an account](https://app.tavily.com/home) and get an API key to use the Tavily search tool.
+
+The Tavily search tool allows you to search the web for information. It is built on top of the [Tavily API](https://tavily.com/).
+
+### Installation
+
+To use [`tavily_search_tool`][pydantic_ai.common_tools.tavily.tavily_search_tool], you need to install
+[`pydantic-ai-slim`](install.md#slim-install) with the `tavily` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[tavily]"
+```
+
+### Usage
+
+Here's an example of how you can use the Tavily search tool with an agent:
+
+```py {title="tavily_search.py" test="skip"}
+import os
+
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.tavily import tavily_search_tool
+
+api_key = os.getenv('TAVILY_API_KEY')
+assert api_key is not None
+
+agent = Agent(
+    'openai:gpt-5.2',
+    tools=[tavily_search_tool(api_key)],
+    instructions='Search Tavily for the given query and return the results.',
+)
+
+result = agent.run_sync('Tell me the top news in the GenAI world, give me links.')
+print(result.output)
+"""
+Here are some of the top recent news articles related to GenAI:
+
+1. How CLEAR users can improve risk analysis with GenAI – Thomson Reuters
+   Read more: https://legal.thomsonreuters.com/blog/how-clear-users-can-improve-risk-analysis-with-genai/
+   (This article discusses how CLEAR's new GenAI-powered tool streamlines risk analysis by quickly summarizing key information from various public data sources.)
+
+2. TELUS Digital Survey Reveals Enterprise Employees Are Entering Sensitive Data Into AI Assistants More Than You Think – FT.com
+   Read more: https://markets.ft.com/data/announce/detail?dockey=600-202502260645BIZWIRE_USPRX____20250226_BW490609-1
+   (This news piece highlights findings from a TELUS Digital survey showing that many enterprise employees use public GenAI tools and sometimes even enter sensitive data.)
+
+3. The Essential Guide to Generative AI – Virtualization Review
+   Read more: https://virtualizationreview.com/Whitepapers/2025/02/SNOWFLAKE-The-Essential-Guide-to-Generative-AI.aspx
+   (This guide provides insights into how GenAI is revolutionizing enterprise strategies and productivity, with input from industry leaders.)
+
+Feel free to click on the links to dive deeper into each story!
+"""
+```
+
+### Configuring Parameters
+
+The `tavily_search_tool` factory accepts optional parameters that control search behavior. `max_results` is always developer-controlled and never appears in the LLM tool schema. Other parameters, when provided, are fixed for all searches and hidden from the LLM's tool schema. Parameters left unset remain available for the LLM to set per-call.
+
+For example, you can lock in `max_results` and `include_domains` at tool creation time while still letting the LLM control `exclude_domains`:
+
+```py {title="tavily_domain_filtering.py"}
+import os
+
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.tavily import tavily_search_tool
+
+api_key = os.getenv('TAVILY_API_KEY')
+assert api_key is not None
+
+agent = Agent(
+    'openai:gpt-5.2',
+    tools=[tavily_search_tool(api_key, max_results=5, include_domains=['arxiv.org'])],
+    instructions='Search for information and return the results.',
+)
+
+result = agent.run_sync(
+    'Find recent papers about transformer architectures'
+)
+print(result.output)
+"""
+Here are some recent papers about transformer architectures from arxiv.org:
+
+1. "Attention Is All You Need" - The foundational paper on the Transformer model.
+2. "FlashAttention: Fast and Memory-Efficient Exact Attention" - Proposes an IO-aware attention algorithm.
+"""
+```
+
+## Exa Search Tool
+
+!!! info
+    Exa is a paid service with free credits to explore their product.
+
+    You need to [sign up for an account](https://dashboard.exa.ai) and get an API key to use the Exa tools.
+
+Exa is a neural search engine that finds high-quality, relevant results across billions of web pages.
+It provides several tools including web search, finding similar pages, content retrieval, and AI-powered answers.
+
+### Installation
+
+To use Exa tools, you need to install [`pydantic-ai-slim`](install.md#slim-install) with the `exa` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[exa]"
+```
+
+### Usage
+
+You can use Exa tools individually or as a toolset. The following tools are available:
+
+- [`exa_search_tool`][pydantic_ai.common_tools.exa.exa_search_tool]: Search the web with various search types (auto, keyword, neural, fast, deep)
+- [`exa_find_similar_tool`][pydantic_ai.common_tools.exa.exa_find_similar_tool]: Find pages similar to a given URL
+- [`exa_get_contents_tool`][pydantic_ai.common_tools.exa.exa_get_contents_tool]: Get full text content from URLs
+- [`exa_answer_tool`][pydantic_ai.common_tools.exa.exa_answer_tool]: Get AI-powered answers with citations
+
+#### Using Individual Tools
+
+```py {title="exa_search.py" test="skip"}
+import os
+
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.exa import exa_search_tool
+
+api_key = os.getenv('EXA_API_KEY')
+assert api_key is not None
+
+agent = Agent(
+    'openai:gpt-5.2',
+    tools=[exa_search_tool(api_key, num_results=5, max_characters=1000)],
+    system_prompt='Search the web for information using Exa.',
+)
+
+result = agent.run_sync('What are the latest developments in quantum computing?')
+print(result.output)
+```
+
+#### Using ExaToolset
+
+For better efficiency when using multiple Exa tools, use [`ExaToolset`][pydantic_ai.common_tools.exa.ExaToolset]
+which shares a single API client across all tools. You can configure which tools to include:
+
+```py {title="exa_toolset.py" test="skip"}
+import os
+
+from pydantic_ai import Agent
+from pydantic_ai.common_tools.exa import ExaToolset
+
+api_key = os.getenv('EXA_API_KEY')
+assert api_key is not None
+
+toolset = ExaToolset(
+    api_key,
+    num_results=5,
+    max_characters=1000,  # Limit text content to control token usage
+    include_search=True,  # Include the search tool (default: True)
+    include_find_similar=True,  # Include the find_similar tool (default: True)
+    include_get_contents=False,  # Exclude the get_contents tool
+    include_answer=True,  # Include the answer tool (default: True)
+)
+
+agent = Agent(
+    'openai:gpt-5.2',
+    toolsets=[toolset],
+    system_prompt='You have access to Exa search tools to find information on the web.',
+)
+
+result = agent.run_sync('Find recent AI research papers and summarize the key findings.')
+print(result.output)
+```

docs/contributing.md

@@ -0,0 +1,154 @@
+We'd love you to contribute to Pydantic AI!
+
+## How we work — the short version
+
+Pydantic AI is maintained by a small team. We set our own priorities based on what benefits the most users, and we work through issues and PRs in that order — not in the order they arrive.
+
+- **Found a bug?** Open an issue with a clear description and a minimal reproducible example. Including a [Logfire](https://logfire.pydantic.dev/) trace link helps us debug dramatically faster.
+- **Want a feature or API change?** Open an issue describing the problem you're solving. Do not start with code.
+- **Want to help build a feature?** Comment on the issue explaining why you need it and what context you bring. We call this being a "champion" — more on that below.
+- **Have a fix or code to share?** Make sure a maintainer has agreed to the approach on the issue and assigned you. Then open a PR.
+
+The rest of this page explains why we work this way and what to expect.
+
+## Before you write code
+
+For anything non-trivial, align with a maintainer on the approach before writing code. A pre-aligned PR is much faster to land than one we're seeing cold.
+
+### Trivial fixes
+
+Typos, broken links, small doc improvements, obvious one-line fixes: just open a PR. No issue needed.
+
+### Bug fixes
+
+If the fix could reasonably go more than one way, or you're unsure it's actually a bug: open an issue first. Include a minimal reproducible example and ideally a [Logfire trace link](https://logfire.pydantic.dev/) showing the problem. For well-scoped bugs, we may generate a fix internally — the most valuable thing you can do is file a clear report and then validate that the fix works for your use case.
+
+### Features, integrations, or API changes
+
+Before writing code, ask whether the change needs to live in core at all. Most new agent behaviors belong in [**Pydantic AI Harness**](https://github.com/pydantic/pydantic-ai-harness), the official capability library — not in this repo. Pydantic AI core is for the agent loop, model providers, and capabilities that require model-specific support or are fundamental to the agent experience. Standalone capabilities — guardrails, memory, context management, file system access, etc. — belong in the harness, where they can iterate faster. See [What goes where?](harness/overview.md#what-goes-where) for the full distinction.
+
+**If your idea is a capability**, open an issue on [pydantic-ai-harness](https://github.com/pydantic/pydantic-ai-harness/issues) instead. You can also publish capabilities as your own package using the `pydantic-ai-<name>` convention — see [Publishing capability packages](extensibility.md#publishing-capability-packages). Once a capability has real users and a stable shape, we can talk about upstreaming to harness or core.
+
+If it does belong in core:
+
+1. **Search first.** If an existing issue covers your need, comment there. If the closest match is only related, open a new issue and link it.
+2. **Describe the problem, not just the solution.** Tell us what you're building, what's blocking you, and what you've tried. This context matters more than code.
+3. **Propose a plan before building.** Post the shape of the solution on the issue, or open a draft PR with just a `PLAN.md`. For larger features, we do short video calls with contributors to iterate on the design — a 20-minute call often saves weeks of async review cycles.
+4. **Wait for assignment.** A maintainer needs to agree on the approach and assign the issue to you before you open a PR. Unassigned PRs may be auto-closed.
+
+!!! warning
+    Writing a large feature PR without prior alignment is the most common way for a contribution to stall or be closed.
+
+## Champions
+
+A "champion" is someone who needs a feature, has context on the problem, and is willing to invest time to help us get it right. If you want to champion a feature:
+
+- Comment on the issue explaining: what you're building, why you need this, and what you can contribute (domain knowledge, testing, validation).
+- We prioritize features where one or more champions with production use cases have stepped up. A feature with no champion stays in the backlog until either we prioritize it ourselves or someone with real context shows up.
+- Being a champion doesn't mean writing the code. It means shaping the plan and validating the result. For significant features, we'll set up a call to iterate on the design together.
+
+Champions are credited as co-authors when the feature ships.
+
+## What to expect during review
+
+### We review PRs in our priority order, not submission order
+
+We do not automatically triage every new PR. PRs on issues we have not pre-aligned on are not in our review queue, regardless of how well written they are. If no maintainer has agreed to the change on an issue and assigned it to you, assume we have not seen your PR.
+
+Even for PRs with code we've previously engaged with: we treat all contributed code as a starting point, not a finished product. We review and prioritize PRs based on the feature's importance to the project, not on how much effort went into the code. This is a change from how open source traditionally worked, and we'd rather be honest about it than leave PRs sitting with no signal.
+
+**If you want to know where your PR stands**, the best thing to do is ping `#pydantic-ai` on [Pydantic Slack](https://logfire.pydantic.dev/docs/join-slack/).
+
+### We may rewrite or supersede your code
+
+We treat contributed code as illustrative: a starting point that shows the shape of the change and proves the approach works, not the final form we merge. The most useful thing you can give us for a non-trivial change is a plan plus a working example — not a polished, merge-ready implementation.
+
+On any PR, we may push commits to your branch, open a follow-up PR that supersedes yours, or rewrite from scratch. For security reasons, we lean toward rewriting contributed code rather than merging as-is. You will still be credited as the original author.
+
+Please don't spend effort chasing green CI, addressing every automated review comment, or rebasing for merge conflicts on a PR we haven't pre-aligned on. If we take the change forward, that polish gets thrown away when we rewrite. Get the approach working, then stop and ping us on Slack.
+
+### Automated review is advisory, not a gate
+
+PRs are automatically reviewed by Devin and our own tooling. These reviews are advisory:
+
+- A bot approval does not mean your PR is ready to merge. Only a human maintainer's review counts.
+- A bot finding does not mean you must act on it. If you disagree, say so.
+- If automated review is generating noise on your PR, tell us. We use that feedback to retune the tooling.
+
+### Priority
+
+We receive far more contributions than we can review, and we focus where it has the most impact. We cannot promise to get to every PR, even good ones, and we'd rather say so up front than leave your work open indefinitely with no signal.
+
+How we weigh priorities:
+
+- **User demand** -- features that more users need get priority. Champion-backed features with production use cases outrank speculative additions.
+- **Provider significance** -- work that affects frontier providers (Anthropic, OpenAI, Google) or providers we know are heavily used gets priority. A model integration for a niche provider will wait; a fix for Anthropic won't.
+- **Roadmap alignment** -- features that align with our current focus areas get priority. Right now that includes the capabilities/hooks API, provider-adaptive tools, and the [Pydantic AI Harness](https://github.com/pydantic/pydantic-ai-harness) capability library.
+- **Capabilities over core** -- features that could live as a [capability](capabilities.md) should go to [Pydantic AI Harness](https://github.com/pydantic/pydantic-ai-harness) or ship as your own package — that's often the fastest path. Once it has traction, come back and we can talk about upstreaming.
+
+## If your PR or issue has gone quiet
+
+1. Ping `#pydantic-ai` on [Pydantic Slack](https://logfire.pydantic.dev/docs/join-slack/) with a link.
+2. Say what you need: "Can you take a look?", "I'm blocked — is this on your radar?", or "Should I close this?" are all fine.
+3. If you've been waiting weeks without any human response, flag it. That's a process failure on our side and we want to know.
+
+## Installation and Setup
+
+Clone your fork and cd into the repo directory
+
+```bash
+git clone git@github.com:<your username>/pydantic-ai.git
+cd pydantic-ai
+```
+
+Install `uv` (version 0.4.30 or later) and `pre-commit`:
+
+- [`uv` install docs](https://docs.astral.sh/uv/getting-started/installation/)
+- [`pre-commit` install docs](https://pre-commit.com/#install)
+
+To install `pre-commit` you can run the following command:
+
+```bash
+uv tool install pre-commit
+```
+
+Install `pydantic-ai`, all dependencies and pre-commit hooks
+
+```bash
+make install
+```
+
+## Running Tests etc.
+
+We use `make` to manage most commands you'll need to run.
+
+For details on available commands, run:
+
+```bash
+make help
+```
+
+To run code formatting, linting, static type checks, and tests with coverage report generation, run:
+
+```bash
+make
+```
+
+## Documentation Changes
+
+To run the documentation page locally, run:
+
+```bash
+uv run mkdocs serve
+```
+
+## Rules for adding new models to Pydantic AI {#new-model-rules}
+
+To avoid an excessive workload for the maintainers of Pydantic AI, we can't accept all model contributions, so we're setting the following rules for when we'll accept new models and when we won't. This should hopefully reduce the chances of disappointment and wasted work.
+
+- To add a new model with an extra dependency, that dependency needs > 500k monthly downloads from PyPI consistently over 3 months or more
+- To add a new model which uses another models logic internally and has no extra dependencies, that model's GitHub org needs > 20k stars in total
+- For any other model that's just a custom URL and API key, we're happy to add a one-paragraph description with a link and instructions on the URL to use
+- For any other model that requires more logic, we recommend you release your own Python package `pydantic-ai-xxx`, which depends on [`pydantic-ai-slim`](install.md#slim-install) and implements a model that inherits from our [`Model`][pydantic_ai.models.Model] ABC
+
+If you're unsure about adding a model, please [create an issue](https://github.com/pydantic/pydantic-ai/issues).

docs/deferred-tools.md

@@ -0,0 +1,475 @@
+# Deferred Tools
+
+There are a few scenarios where the model should be able to call a tool that should not or cannot be executed during the same agent run inside the same Python process:
+
+- it may need to be approved by the user first
+- it may depend on an upstream service, frontend, or user to provide the result
+- the result could take longer to generate than it's reasonable to keep the agent process running
+
+To support these use cases, Pydantic AI provides the concept of deferred tools, which come in two flavors documented below:
+
+- tools that [require approval](#human-in-the-loop-tool-approval)
+- tools that are [executed externally](#external-tool-execution)
+
+When the model calls a deferred tool, there are two ways to resolve it:
+
+- **Resolve it inline**, using a [`HandleDeferredToolCalls`][pydantic_ai.capabilities.HandleDeferredToolCalls] [capability](capabilities.md) with a handler that resolves some or all of the pending calls. The agent run continues in a single call without needing to end and restart — use this when the resolver (e.g. an approval gate, an external service client) lives in the same process as the agent. See [Resolving deferred calls with a handler](#resolving-deferred-calls-with-a-handler).
+- **End the run** with a [`DeferredToolRequests`][pydantic_ai.tools.DeferredToolRequests] output object containing information about the deferred tool calls; the caller gathers approvals/results and then starts a new agent run with the original run's [message history](message-history.md) plus a [`DeferredToolResults`][pydantic_ai.tools.DeferredToolResults] object. Use this when the resolver lives outside the agent process — e.g. a UI adapter that surfaces pending calls to a user and starts a follow-up run once it has their response.
+
+The two flows compose: a handler can resolve a subset of calls and let the rest bubble up as `DeferredToolRequests` output for an outer caller to handle.
+
+The stop-the-world flow requires `DeferredToolRequests` to be in the `Agent`'s [`output_type`](output.md#structured-output) so that the possible types of the agent run output are correctly inferred. If your agent can also be used in a context where no deferred tools are available and you don't want to deal with that type everywhere you use the agent, you can instead pass the `output_type` argument when you run the agent using [`agent.run()`][pydantic_ai.agent.AbstractAgent.run], [`agent.run_sync()`][pydantic_ai.agent.AbstractAgent.run_sync], [`agent.run_stream()`][pydantic_ai.agent.AbstractAgent.run_stream], or [`agent.iter()`][pydantic_ai.agent.Agent.iter]. Note that the run-time `output_type` overrides the one specified at construction time (for type inference reasons), so you'll need to include the original output type explicitly.
+
+## Resolving deferred calls with a handler
+
+The recommended way to handle deferred tool calls is to register a [`HandleDeferredToolCalls`][pydantic_ai.capabilities.HandleDeferredToolCalls] [capability](capabilities.md) whose handler receives the [`DeferredToolRequests`][pydantic_ai.tools.DeferredToolRequests] and returns a [`DeferredToolResults`][pydantic_ai.tools.DeferredToolResults] resolving some or all of them. The tool execution pipeline applies the results inline and the agent run continues in a single call, as if the deferred tools had returned normally.
+
+With a handler in place, `DeferredToolRequests` no longer needs to be declared as an output type — unless you also want unresolved calls to bubble up to the caller (see below).
+
+[`DeferredToolRequests.build_results()`][pydantic_ai.tools.DeferredToolRequests.build_results] is a convenience constructor — it validates that every tool call ID refers to a pending request of the correct kind, and accepts `approve_all=True` to auto-approve any approval requests not otherwise specified.
+
+```python {title="deferred_tool_handler.py"}
+from pydantic_ai import (
+    Agent,
+    ApprovalRequired,
+    CallDeferred,
+    DeferredToolRequests,
+    DeferredToolResults,
+    RunContext,
+    ToolDenied,
+)
+from pydantic_ai.capabilities import HandleDeferredToolCalls
+
+
+async def handle_deferred(
+    ctx: RunContext, requests: DeferredToolRequests
+) -> DeferredToolResults:
+    approvals: dict[str, bool | ToolDenied] = {}
+    for call in requests.approvals:
+        if call.tool_name == 'delete_file':
+            approvals[call.tool_call_id] = ToolDenied('Deleting files is not allowed')
+        else:
+            approvals[call.tool_call_id] = True
+
+    calls = {call.tool_call_id: f'(external result for {call.tool_name})' for call in requests.calls}
+
+    return requests.build_results(approvals=approvals, calls=calls)
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    capabilities=[HandleDeferredToolCalls(handler=handle_deferred)],
+)
+
+
+@agent.tool_plain(requires_approval=True)
+def delete_file(path: str) -> str:
+    return f'File {path!r} deleted'  # (1)!
+
+
+@agent.tool
+def update_file(ctx: RunContext, path: str) -> str:
+    if path == '.env' and not ctx.tool_call_approved:
+        raise ApprovalRequired
+    return f'File {path!r} updated'
+
+
+@agent.tool_plain
+async def send_to_worker(task: str) -> str:
+    raise CallDeferred  # (2)!
+```
+
+1. Never reached here — the handler denies this call, so the model sees the denial message instead.
+2. The handler supplies the result for this external call, so the tool body just signals the deferral.
+
+If the handler declines to resolve some or all of the calls (by omitting them from the returned [`DeferredToolResults`][pydantic_ai.tools.DeferredToolResults] or returning `None`), the next [`HandleDeferredToolCalls`][pydantic_ai.capabilities.HandleDeferredToolCalls] (or any other capability that overrides the [`handle_deferred_tool_calls`][pydantic_ai.capabilities.AbstractCapability.handle_deferred_tool_calls] hook) gets a chance, and any still-unresolved calls bubble up as a [`DeferredToolRequests`][pydantic_ai.tools.DeferredToolRequests] output. To allow that bubble-up, include `DeferredToolRequests` in the agent's `output_type` — so you can combine inline handling with the stop-the-world flow when it makes sense.
+
+If you're [building a custom capability](capabilities.md#building-custom-capabilities) that needs to resolve approvals or external calls itself (e.g. a sandbox that exposes deferred tools), override the [`handle_deferred_tool_calls`][pydantic_ai.capabilities.AbstractCapability.handle_deferred_tool_calls] hook directly on your capability instead of registering a separate `HandleDeferredToolCalls`. The same hook is also available via the [`Hooks`][pydantic_ai.capabilities.Hooks] capability — see [Hooks](hooks.md#deferred-tool-call-hook).
+
+The sections below describe the two kinds of deferred tools the handler can resolve, as well as the alternative stop-the-world flow for each. See [Capabilities](capabilities.md) for how multiple capabilities compose, including [`WrapperCapability`][pydantic_ai.capabilities.WrapperCapability] and the `capabilities=[...]` list.
+
+## Human-in-the-Loop Tool Approval
+
+If a tool function always requires approval, you can pass the `requires_approval=True` argument to the [`@agent.tool`][pydantic_ai.agent.Agent.tool] decorator, [`@agent.tool_plain`][pydantic_ai.agent.Agent.tool_plain] decorator, [`Tool`][pydantic_ai.tools.Tool] class, [`FunctionToolset.tool`][pydantic_ai.toolsets.FunctionToolset.tool] decorator, or [`FunctionToolset.add_function()`][pydantic_ai.toolsets.FunctionToolset.add_function] method. Inside the function, you can then assume that the tool call has been approved.
+
+If whether a tool function requires approval depends on the tool call arguments or the agent [run context][pydantic_ai.tools.RunContext] (e.g. [dependencies](dependencies.md) or message history), you can raise the [`ApprovalRequired`][pydantic_ai.exceptions.ApprovalRequired] exception from the tool function. The [`RunContext.tool_call_approved`][pydantic_ai.tools.RunContext.tool_call_approved] property will be `True` if the tool call has already been approved.
+
+To require approval for calls to tools provided by a [toolset](toolsets.md) (like an [MCP server](mcp/client.md)), see the [`ApprovalRequiredToolset` documentation](toolsets.md#requiring-tool-approval).
+
+When the model calls a tool that requires approval, the agent run will end with a [`DeferredToolRequests`][pydantic_ai.tools.DeferredToolRequests] output object with an `approvals` list holding [`ToolCallPart`s][pydantic_ai.messages.ToolCallPart] containing the tool name, validated arguments, and a unique tool call ID.
+
+Once you've gathered the user's approvals or denials, you can build a [`DeferredToolResults`][pydantic_ai.tools.DeferredToolResults] object with an `approvals` dictionary that maps each tool call ID to a boolean, a [`ToolApproved`][pydantic_ai.tools.ToolApproved] object (with optional `override_args`), or a [`ToolDenied`][pydantic_ai.tools.ToolDenied] object (with an optional custom `message` to provide to the model). You can also provide a `metadata` dictionary on `DeferredToolResults` that maps each tool call ID to a dictionary of metadata that will be available in the tool's [`RunContext.tool_call_metadata`][pydantic_ai.tools.RunContext.tool_call_metadata] attribute. This `DeferredToolResults` object can then be provided to one of the agent run methods as `deferred_tool_results`, alongside the original run's [message history](message-history.md).
+
+Here's an example that shows how to require approval for all file deletions, and for updates of specific protected files:
+
+```python {title="tool_requires_approval.py"}
+from pydantic_ai import (
+    Agent,
+    ApprovalRequired,
+    DeferredToolRequests,
+    DeferredToolResults,
+    RunContext,
+    ToolDenied,
+)
+
+agent = Agent('openai:gpt-5.2', output_type=[str, DeferredToolRequests])
+
+PROTECTED_FILES = {'.env'}
+
+
+@agent.tool
+def update_file(ctx: RunContext, path: str, content: str) -> str:
+    if path in PROTECTED_FILES and not ctx.tool_call_approved:
+        raise ApprovalRequired(metadata={'reason': 'protected'})  # (1)!
+    return f'File {path!r} updated: {content!r}'
+
+
+@agent.tool_plain(requires_approval=True)
+def delete_file(path: str) -> str:
+    return f'File {path!r} deleted'
+
+
+result = agent.run_sync('Delete `__init__.py`, write `Hello, world!` to `README.md`, and clear `.env`')
+messages = result.all_messages()
+
+assert isinstance(result.output, DeferredToolRequests)
+requests = result.output
+print(requests)
+"""
+DeferredToolRequests(
+    calls=[],
+    approvals=[
+        ToolCallPart(
+            tool_name='update_file',
+            args={'path': '.env', 'content': ''},
+            tool_call_id='update_file_dotenv',
+        ),
+        ToolCallPart(
+            tool_name='delete_file',
+            args={'path': '__init__.py'},
+            tool_call_id='delete_file',
+        ),
+    ],
+    metadata={'update_file_dotenv': {'reason': 'protected'}},
+)
+"""
+
+results = DeferredToolResults()
+for call in requests.approvals:
+    result = False
+    if call.tool_name == 'update_file':
+        # Approve all updates
+        result = True
+    elif call.tool_name == 'delete_file':
+        # deny all deletes
+        result = ToolDenied('Deleting files is not allowed')
+
+    results.approvals[call.tool_call_id] = result
+
+result = agent.run_sync(
+    'Now create a backup of README.md',  # (2)!
+    message_history=messages,
+    deferred_tool_results=results,
+)
+print(result.output)
+"""
+Here's what I've done:
+- Attempted to delete __init__.py, but deletion is not allowed.
+- Updated README.md with: Hello, world!
+- Cleared .env (set to empty).
+- Created a backup at README.md.bak containing: Hello, world!
+
+If you want a different backup name or format (e.g., timestamped like README_2025-11-24.bak), let me know.
+"""
+print(result.all_messages())
+"""
+[
+    ModelRequest(
+        parts=[
+            UserPromptPart(
+                content='Delete `__init__.py`, write `Hello, world!` to `README.md`, and clear `.env`',
+                timestamp=datetime.datetime(...),
+            )
+        ],
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelResponse(
+        parts=[
+            ToolCallPart(
+                tool_name='delete_file',
+                args={'path': '__init__.py'},
+                tool_call_id='delete_file',
+            ),
+            ToolCallPart(
+                tool_name='update_file',
+                args={'path': 'README.md', 'content': 'Hello, world!'},
+                tool_call_id='update_file_readme',
+            ),
+            ToolCallPart(
+                tool_name='update_file',
+                args={'path': '.env', 'content': ''},
+                tool_call_id='update_file_dotenv',
+            ),
+        ],
+        usage=RequestUsage(input_tokens=63, output_tokens=21),
+        model_name='gpt-5.2',
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelRequest(
+        parts=[
+            ToolReturnPart(
+                tool_name='update_file',
+                content="File 'README.md' updated: 'Hello, world!'",
+                tool_call_id='update_file_readme',
+                timestamp=datetime.datetime(...),
+            )
+        ],
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelRequest(
+        parts=[
+            ToolReturnPart(
+                tool_name='delete_file',
+                content='Deleting files is not allowed',
+                tool_call_id='delete_file',
+                timestamp=datetime.datetime(...),
+                outcome='denied',
+            ),
+            ToolReturnPart(
+                tool_name='update_file',
+                content="File '.env' updated: ''",
+                tool_call_id='update_file_dotenv',
+                timestamp=datetime.datetime(...),
+            ),
+            UserPromptPart(
+                content='Now create a backup of README.md',
+                timestamp=datetime.datetime(...),
+            ),
+        ],
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelResponse(
+        parts=[
+            ToolCallPart(
+                tool_name='update_file',
+                args={'path': 'README.md.bak', 'content': 'Hello, world!'},
+                tool_call_id='update_file_backup',
+            )
+        ],
+        usage=RequestUsage(input_tokens=86, output_tokens=31),
+        model_name='gpt-5.2',
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelRequest(
+        parts=[
+            ToolReturnPart(
+                tool_name='update_file',
+                content="File 'README.md.bak' updated: 'Hello, world!'",
+                tool_call_id='update_file_backup',
+                timestamp=datetime.datetime(...),
+            )
+        ],
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+    ModelResponse(
+        parts=[
+            TextPart(
+                content="Here's what I've done:\n- Attempted to delete __init__.py, but deletion is not allowed.\n- Updated README.md with: Hello, world!\n- Cleared .env (set to empty).\n- Created a backup at README.md.bak containing: Hello, world!\n\nIf you want a different backup name or format (e.g., timestamped like README_2025-11-24.bak), let me know."
+            )
+        ],
+        usage=RequestUsage(input_tokens=93, output_tokens=89),
+        model_name='gpt-5.2',
+        timestamp=datetime.datetime(...),
+        run_id='...',
+        conversation_id='...',
+    ),
+]
+"""
+```
+
+1. The optional `metadata` parameter can attach arbitrary context to deferred tool calls, accessible in `DeferredToolRequests.metadata` keyed by `tool_call_id`.
+2. This second agent run continues from where the first run left off, providing the tool approval results and optionally a new `user_prompt` to give the model additional instructions alongside the deferred results.
+
+_(This example is complete, it can be run "as is")_
+
+!!! note "Tool result ordering"
+    Tool results follow the order in which the model emitted the corresponding tool calls. In the message history above, `delete_file`'s denied result appears before `update_file`'s result for `.env` because the model emitted `delete_file` first. This is an intentional behavior change in v2: results are no longer grouped by tool kind, so the ordering you see reflects the model's emission order.
+
+## External Tool Execution
+
+When the result of a tool call cannot be generated inside the same agent run in which it was called, the tool is considered to be external.
+Examples of external tools are client-side tools implemented by a web or app frontend, and slow tasks that are passed off to a background worker or external service instead of keeping the agent process running.
+
+If whether a tool call should be executed externally depends on the tool call arguments, the agent [run context][pydantic_ai.tools.RunContext] (e.g. [dependencies](dependencies.md) or message history), or how long the task is expected to take, you can define a tool function and conditionally raise the [`CallDeferred`][pydantic_ai.exceptions.CallDeferred] exception. Before raising the exception, the tool function would typically schedule some background task and pass along the [`RunContext.tool_call_id`][pydantic_ai.tools.RunContext.tool_call_id] so that the result can be matched to the deferred tool call later.
+
+If a tool is always executed externally and its definition is provided to your code along with a JSON schema for its arguments, you can use an [`ExternalToolset`](toolsets.md#external-toolset). If the external tools are known up front and you don't have the arguments JSON schema handy, you can also define a tool function with the appropriate signature that does nothing but raise the [`CallDeferred`][pydantic_ai.exceptions.CallDeferred] exception.
+
+When the model calls an external tool, the agent run will end with a [`DeferredToolRequests`][pydantic_ai.tools.DeferredToolRequests] output object with a `calls` list holding [`ToolCallPart`s][pydantic_ai.messages.ToolCallPart] containing the tool name, validated arguments, and a unique tool call ID.
+
+Once the tool call results are ready, you can build a [`DeferredToolResults`][pydantic_ai.tools.DeferredToolResults] object with a `calls` dictionary that maps each tool call ID to an arbitrary value to be returned to the model, a [`ToolReturn`](tools-advanced.md#advanced-tool-returns) object, or a [`ModelRetry`][pydantic_ai.exceptions.ModelRetry] exception in case the tool call failed and the model should [try again](tools-advanced.md#tool-retries). This `DeferredToolResults` object can then be provided to one of the agent run methods as `deferred_tool_results`, alongside the original run's [message history](message-history.md).
+
+Here's an example that shows how to move a task that takes a while to complete to the background and return the result to the model once the task is complete:
+
+```python {title="external_tool.py"}
+import asyncio
+from dataclasses import dataclass
+from typing import Any
+
+from pydantic_ai import (
+    Agent,
+    CallDeferred,
+    DeferredToolRequests,
+    DeferredToolResults,
+    ModelRetry,
+    RunContext,
+)
+
+
+@dataclass
+class TaskResult:
+    task_id: str
+    result: Any
+
+
+async def calculate_answer_task(task_id: str, question: str) -> TaskResult:
+    await asyncio.sleep(1)
+    return TaskResult(task_id=task_id, result=42)
+
+
+agent = Agent('openai:gpt-5.2', output_type=[str, DeferredToolRequests])
+
+tasks: list[asyncio.Task[TaskResult]] = []
+
+
+@agent.tool
+async def calculate_answer(ctx: RunContext, question: str) -> str:
+    task_id = f'task_{len(tasks)}'  # (1)!
+    task = asyncio.create_task(calculate_answer_task(task_id, question))
+    tasks.append(task)
+
+    raise CallDeferred(metadata={'task_id': task_id})  # (2)!
+
+
+async def main():
+    result = await agent.run('Calculate the answer to the ultimate question of life, the universe, and everything')
+    messages = result.all_messages()
+
+    assert isinstance(result.output, DeferredToolRequests)
+    requests = result.output
+    print(requests)
+    """
+    DeferredToolRequests(
+        calls=[
+            ToolCallPart(
+                tool_name='calculate_answer',
+                args={
+                    'question': 'the ultimate question of life, the universe, and everything'
+                },
+                tool_call_id='pyd_ai_tool_call_id',
+            )
+        ],
+        approvals=[],
+        metadata={'pyd_ai_tool_call_id': {'task_id': 'task_0'}},
+    )
+    """
+
+    done, _ = await asyncio.wait(tasks)  # (3)!
+    task_results = [task.result() for task in done]
+    task_results_by_task_id = {result.task_id: result.result for result in task_results}
+
+    results = DeferredToolResults()
+    for call in requests.calls:
+        try:
+            task_id = requests.metadata[call.tool_call_id]['task_id']
+            result = task_results_by_task_id[task_id]
+        except KeyError:
+            result = ModelRetry('No result for this tool call was found.')
+
+        results.calls[call.tool_call_id] = result
+
+    result = await agent.run(message_history=messages, deferred_tool_results=results)
+    print(result.output)
+    #> The answer to the ultimate question of life, the universe, and everything is 42.
+    print(result.all_messages())
+    """
+    [
+        ModelRequest(
+            parts=[
+                UserPromptPart(
+                    content='Calculate the answer to the ultimate question of life, the universe, and everything',
+                    timestamp=datetime.datetime(...),
+                )
+            ],
+            timestamp=datetime.datetime(...),
+            run_id='...',
+            conversation_id='...',
+        ),
+        ModelResponse(
+            parts=[
+                ToolCallPart(
+                    tool_name='calculate_answer',
+                    args={
+                        'question': 'the ultimate question of life, the universe, and everything'
+                    },
+                    tool_call_id='pyd_ai_tool_call_id',
+                )
+            ],
+            usage=RequestUsage(input_tokens=63, output_tokens=13),
+            model_name='gpt-5.2',
+            timestamp=datetime.datetime(...),
+            run_id='...',
+            conversation_id='...',
+        ),
+        ModelRequest(
+            parts=[
+                ToolReturnPart(
+                    tool_name='calculate_answer',
+                    content=42,
+                    tool_call_id='pyd_ai_tool_call_id',
+                    timestamp=datetime.datetime(...),
+                )
+            ],
+            timestamp=datetime.datetime(...),
+            run_id='...',
+            conversation_id='...',
+        ),
+        ModelResponse(
+            parts=[
+                TextPart(
+                    content='The answer to the ultimate question of life, the universe, and everything is 42.'
+                )
+            ],
+            usage=RequestUsage(input_tokens=64, output_tokens=28),
+            model_name='gpt-5.2',
+            timestamp=datetime.datetime(...),
+            run_id='...',
+            conversation_id='...',
+        ),
+    ]
+    """
+```
+
+1. Generate a task ID that can be tracked independently of the tool call ID.
+2. The optional `metadata` parameter passes the `task_id` so it can be matched with results later, accessible in `DeferredToolRequests.metadata` keyed by `tool_call_id`.
+3. In reality, this would typically happen in a separate process that polls for the task status or is notified when all pending tasks are complete.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## See Also
+
+- [Function Tools](tools.md) - Basic tool concepts and registration
+- [Advanced Tool Features](tools-advanced.md) - Custom schemas, dynamic tools, and execution details
+- [Toolsets](toolsets.md) - Managing collections of tools, including `ExternalToolset` for external tools
+- [Message History](message-history.md) - Understanding how to work with message history for deferred tools

docs/dependencies.md

@@ -0,0 +1,310 @@
+# Dependencies
+
+Pydantic AI uses a dependency injection system to provide data and services to your agent's [system prompts](agent.md#system-prompts), [tools](tools.md) and [output validators](output.md#output-validator-functions).
+
+Matching Pydantic AI's design philosophy, our dependency system tries to use existing best practice in Python development rather than inventing esoteric "magic", this should make dependencies type-safe, understandable, easier to test, and ultimately easier to deploy in production.
+
+## Defining Dependencies
+
+Dependencies can be any python type. While in simple cases you might be able to pass a single object as a dependency (e.g. an HTTP connection), [dataclasses][] are generally a convenient container when your dependencies included multiple objects.
+
+Here's an example of defining an agent that requires dependencies.
+
+(**Note:** dependencies aren't actually used in this example, see [Accessing Dependencies](#accessing-dependencies) below)
+
+```python {title="unused_dependencies.py"}
+from dataclasses import dataclass
+
+import httpx
+
+from pydantic_ai import Agent
+
+
+@dataclass
+class MyDeps:  # (1)!
+    api_key: str
+    http_client: httpx.AsyncClient
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    deps_type=MyDeps,  # (2)!
+)
+
+
+async def main():
+    async with httpx.AsyncClient() as client:
+        deps = MyDeps('foobar', client)
+        result = await agent.run(
+            'Tell me a joke.',
+            deps=deps,  # (3)!
+        )
+        print(result.output)
+        #> Did you hear about the toothpaste scandal? They called it Colgate.
+```
+
+1. Define a dataclass to hold dependencies.
+2. Pass the dataclass type to the `deps_type` argument of the [`Agent` constructor][pydantic_ai.agent.Agent.__init__]. **Note**: we're passing the type here, NOT an instance, this parameter is not actually used at runtime, it's here so we can get full type checking of the agent.
+3. When running the agent, pass an instance of the dataclass to the `deps` parameter.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Accessing Dependencies
+
+Dependencies are accessed through the [`RunContext`][pydantic_ai.tools.RunContext] type, this should be the first parameter of system prompt functions etc.
+
+```python {title="system_prompt_dependencies.py" hl_lines="20-27"}
+from dataclasses import dataclass
+
+import httpx
+
+from pydantic_ai import Agent, RunContext
+
+
+@dataclass
+class MyDeps:
+    api_key: str
+    http_client: httpx.AsyncClient
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    deps_type=MyDeps,
+)
+
+
+@agent.system_prompt  # (1)!
+async def get_system_prompt(ctx: RunContext[MyDeps]) -> str:  # (2)!
+    response = await ctx.deps.http_client.get(  # (3)!
+        'https://example.com',
+        headers={'Authorization': f'Bearer {ctx.deps.api_key}'},  # (4)!
+    )
+    response.raise_for_status()
+    return f'Prompt: {response.text}'
+
+
+async def main():
+    async with httpx.AsyncClient() as client:
+        deps = MyDeps('foobar', client)
+        result = await agent.run('Tell me a joke.', deps=deps)
+        print(result.output)
+        #> Did you hear about the toothpaste scandal? They called it Colgate.
+```
+
+1. [`RunContext`][pydantic_ai.tools.RunContext] may optionally be passed to a [`system_prompt`][pydantic_ai.agent.Agent.system_prompt] function as the only argument.
+2. [`RunContext`][pydantic_ai.tools.RunContext] is parameterized with the type of the dependencies, if this type is incorrect, static type checkers will raise an error.
+3. Access dependencies through the [`.deps`][pydantic_ai.tools.RunContext.deps] attribute.
+4. Access dependencies through the [`.deps`][pydantic_ai.tools.RunContext.deps] attribute.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+In addition to [`.deps`][pydantic_ai.tools.RunContext.deps], [`RunContext`][pydantic_ai.tools.RunContext] provides access to the running agent via [`.agent`][pydantic_ai.tools.RunContext.agent], which is useful when [tools](tools.md), [hooks](hooks.md), or [capabilities](capabilities.md) need to read agent properties like [`name`][pydantic_ai.agent.Agent.name] or [`output_type`][pydantic_ai.agent.Agent.output_type].
+
+Dependency fields can also be referenced in instructions and descriptions via [template strings](agent-spec.md#template-strings) — for example, `TemplateStr('Hello {{name}}')` renders `name` from the deps object at runtime. This is especially useful in [agent specs](agent-spec.md) where callables aren't available.
+
+### Asynchronous vs. Synchronous dependencies
+
+[System prompt functions](agent.md#system-prompts), [function tools](tools.md) and [output validators](output.md#output-validator-functions) are all run in the async context of an agent run.
+
+If these functions are not coroutines (e.g. `async def`) they are called with
+[`run_in_executor`][asyncio.loop.run_in_executor] in a thread pool. It's therefore marginally preferable
+to use `async` methods where dependencies perform IO, although synchronous dependencies should work fine too.
+
+!!! note "`run` vs. `run_sync` and Asynchronous vs. Synchronous dependencies"
+    Whether you use synchronous or asynchronous dependencies is completely independent of whether you use `run` or `run_sync` — `run_sync` is just a wrapper around `run` and agents are always run in an async context.
+
+Here's the same example as above, but with a synchronous dependency:
+
+```python {title="sync_dependencies.py"}
+from dataclasses import dataclass
+
+import httpx
+
+from pydantic_ai import Agent, RunContext
+
+
+@dataclass
+class MyDeps:
+    api_key: str
+    http_client: httpx.Client  # (1)!
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    deps_type=MyDeps,
+)
+
+
+@agent.system_prompt
+def get_system_prompt(ctx: RunContext[MyDeps]) -> str:  # (2)!
+    response = ctx.deps.http_client.get(
+        'https://example.com', headers={'Authorization': f'Bearer {ctx.deps.api_key}'}
+    )
+    response.raise_for_status()
+    return f'Prompt: {response.text}'
+
+
+async def main():
+    deps = MyDeps('foobar', httpx.Client())
+    result = await agent.run(
+        'Tell me a joke.',
+        deps=deps,
+    )
+    print(result.output)
+    #> Did you hear about the toothpaste scandal? They called it Colgate.
+```
+
+1. Here we use a synchronous `httpx.Client` instead of an asynchronous `httpx.AsyncClient`.
+2. To match the synchronous dependency, the system prompt function is now a plain function, not a coroutine.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Full Example
+
+As well as system prompts, dependencies can be used in [tools](tools.md) and [output validators](output.md#output-validator-functions).
+
+```python {title="full_example.py" hl_lines="27-35 38-48"}
+from dataclasses import dataclass
+
+import httpx
+
+from pydantic_ai import Agent, ModelRetry, RunContext
+
+
+@dataclass
+class MyDeps:
+    api_key: str
+    http_client: httpx.AsyncClient
+
+
+agent = Agent(
+    'openai:gpt-5.2',
+    deps_type=MyDeps,
+)
+
+
+@agent.system_prompt
+async def get_system_prompt(ctx: RunContext[MyDeps]) -> str:
+    response = await ctx.deps.http_client.get('https://example.com')
+    response.raise_for_status()
+    return f'Prompt: {response.text}'
+
+
+@agent.tool  # (1)!
+async def get_joke_material(ctx: RunContext[MyDeps], subject: str) -> str:
+    response = await ctx.deps.http_client.get(
+        'https://example.com#jokes',
+        params={'subject': subject},
+        headers={'Authorization': f'Bearer {ctx.deps.api_key}'},
+    )
+    response.raise_for_status()
+    return response.text
+
+
+@agent.output_validator  # (2)!
+async def validate_output(ctx: RunContext[MyDeps], output: str) -> str:
+    response = await ctx.deps.http_client.post(
+        'https://example.com#validate',
+        headers={'Authorization': f'Bearer {ctx.deps.api_key}'},
+        params={'query': output},
+    )
+    if response.status_code == 400:
+        raise ModelRetry(f'invalid response: {response.text}')
+    response.raise_for_status()
+    return output
+
+
+async def main():
+    async with httpx.AsyncClient() as client:
+        deps = MyDeps('foobar', client)
+        result = await agent.run('Tell me a joke.', deps=deps)
+        print(result.output)
+        #> Did you hear about the toothpaste scandal? They called it Colgate.
+```
+
+1. To pass `RunContext` to a tool, use the [`tool`][pydantic_ai.agent.Agent.tool] decorator.
+2. `RunContext` may optionally be passed to a [`output_validator`][pydantic_ai.agent.Agent.output_validator] function as the first argument.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Overriding Dependencies
+
+When testing agents, it's useful to be able to customise dependencies.
+
+While this can sometimes be done by calling the agent directly within unit tests, we can also override dependencies
+while calling application code which in turn calls the agent.
+
+This is done via the [`override`][pydantic_ai.agent.Agent.override] method on the agent.
+
+```python {title="joke_app.py"}
+from dataclasses import dataclass
+
+import httpx
+
+from pydantic_ai import Agent, RunContext
+
+
+@dataclass
+class MyDeps:
+    api_key: str
+    http_client: httpx.AsyncClient
+
+    async def system_prompt_factory(self) -> str:  # (1)!
+        response = await self.http_client.get('https://example.com')
+        response.raise_for_status()
+        return f'Prompt: {response.text}'
+
+
+joke_agent = Agent('openai:gpt-5.2', deps_type=MyDeps)
+
+
+@joke_agent.system_prompt
+async def get_system_prompt(ctx: RunContext[MyDeps]) -> str:
+    return await ctx.deps.system_prompt_factory()  # (2)!
+
+
+async def application_code(prompt: str) -> str:  # (3)!
+    ...
+    ...
+    # now deep within application code we call our agent
+    async with httpx.AsyncClient() as client:
+        app_deps = MyDeps('foobar', client)
+        result = await joke_agent.run(prompt, deps=app_deps)  # (4)!
+    return result.output
+```
+
+1. Define a method on the dependency to make the system prompt easier to customise.
+2. Call the system prompt factory from within the system prompt function.
+3. Application code that calls the agent, in a real application this might be an API endpoint.
+4. Call the agent from within the application code, in a real application this call might be deep within a call stack. Note `app_deps` here will NOT be used when deps are overridden.
+
+_(This example is complete, it can be run "as is")_
+
+```python {title="test_joke_app.py" hl_lines="10-12" call_name="test_application_code" requires="joke_app.py"}
+from joke_app import MyDeps, application_code, joke_agent
+
+
+class TestMyDeps(MyDeps):  # (1)!
+    async def system_prompt_factory(self) -> str:
+        return 'test prompt'
+
+
+async def test_application_code():
+    test_deps = TestMyDeps('test_key', None)  # (2)!
+    with joke_agent.override(deps=test_deps):  # (3)!
+        joke = await application_code('Tell me a joke.')  # (4)!
+    assert joke.startswith('Did you hear about the toothpaste scandal?')
+```
+
+1. Define a subclass of `MyDeps` in tests to customise the system prompt factory.
+2. Create an instance of the test dependency, we don't need to pass an `http_client` here as it's not used.
+3. Override the dependencies of the agent for the duration of the `with` block, `test_deps` will be used when the agent is run.
+4. Now we can safely call our application code, the agent will use the overridden dependencies.
+
+## Examples
+
+The following examples demonstrate how to use dependencies in Pydantic AI:
+
+- [Weather Agent](examples/weather-agent.md)
+- [SQL Generation](examples/sql-gen.md)
+- [RAG](examples/rag.md)

docs/direct.md

@@ -0,0 +1,154 @@
+# Direct Model Requests
+
+The `direct` module provides low-level methods for making imperative requests to LLMs where the only abstraction is input and output schema translation, enabling you to use all models with the same API.
+
+These methods are thin wrappers around the [`Model`][pydantic_ai.models.Model] implementations, offering a simpler interface when you don't need the full functionality of an [`Agent`][pydantic_ai.Agent].
+
+The following functions are available:
+
+- [`model_request`][pydantic_ai.direct.model_request]: Make a non-streamed async request to a model
+- [`model_request_sync`][pydantic_ai.direct.model_request_sync]: Make a non-streamed synchronous request to a model
+- [`model_request_stream`][pydantic_ai.direct.model_request_stream]: Make a streamed async request to a model
+- [`model_request_stream_sync`][pydantic_ai.direct.model_request_stream_sync]: Make a streamed sync request to a model
+
+## Basic Example
+
+Here's a simple example demonstrating how to use the direct API to make a basic request:
+
+```python title="direct_basic.py"
+from pydantic_ai import ModelRequest
+from pydantic_ai.direct import model_request_sync
+
+# Make a synchronous request to the model
+model_response = model_request_sync(
+    'anthropic:claude-haiku-4-5',
+    [ModelRequest.user_text_prompt('What is the capital of France?')]
+)
+
+print(model_response.parts[0].content)
+#> The capital of France is Paris.
+print(model_response.usage)
+#> RequestUsage(input_tokens=56, output_tokens=7)
+```
+
+_(This example is complete, it can be run "as is")_
+
+!!! note
+    Instructions are not cumulative across message history. If multiple [`ModelRequest`][pydantic_ai.messages.ModelRequest]s include [`instructions`][pydantic_ai.messages.ModelRequest.instructions], the direct API uses the most recent one.
+
+## Advanced Example with Tool Calling
+
+You can also use the direct API to work with function/tool calling.
+
+Even here we can use Pydantic to generate the JSON schema for the tool:
+
+```python
+from typing import Literal
+
+from pydantic import BaseModel
+
+from pydantic_ai import ModelRequest, ToolDefinition
+from pydantic_ai.direct import model_request
+from pydantic_ai.models import ModelRequestParameters
+
+
+class Divide(BaseModel):
+    """Divide two numbers."""
+
+    numerator: float
+    denominator: float
+    on_inf: Literal['error', 'infinity'] = 'infinity'
+
+
+async def main():
+    # Make a request to the model with tool access
+    model_response = await model_request(
+        'openai:gpt-5-nano',
+        [ModelRequest.user_text_prompt('What is 123 / 456?')],
+        model_request_parameters=ModelRequestParameters(
+            function_tools=[
+                ToolDefinition(
+                    name=Divide.__name__.lower(),
+                    description=Divide.__doc__,
+                    parameters_json_schema=Divide.model_json_schema(),
+                )
+            ],
+            allow_text_output=True,  # Allow model to either use tools or respond directly
+        ),
+    )
+    print(model_response)
+    """
+    ModelResponse(
+        parts=[
+            ToolCallPart(
+                tool_name='divide',
+                args={'numerator': '123', 'denominator': '456'},
+                tool_call_id='pyd_ai_2e0e396768a14fe482df90a29a78dc7b',
+            )
+        ],
+        usage=RequestUsage(input_tokens=55, output_tokens=7),
+        model_name='gpt-5-nano',
+        timestamp=datetime.datetime(...),
+    )
+    """
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## When to Use the direct API vs Agent
+
+The direct API is ideal when:
+
+1. You need more direct control over model interactions
+2. You want to implement custom behavior around model requests
+3. You're building your own abstractions on top of model interactions
+
+For most application use cases, the higher-level [`Agent`][pydantic_ai.Agent] API provides a more convenient interface with additional features such as native tool execution, retrying, structured output parsing, and more.
+
+## OpenTelemetry or Logfire Instrumentation
+
+As with [agents][pydantic_ai.Agent], you can enable OpenTelemetry/Logfire instrumentation with just a few extra lines
+
+```python {title="direct_instrumented.py" hl_lines="1 6 7"}
+import logfire
+
+from pydantic_ai import ModelRequest
+from pydantic_ai.direct import model_request_sync
+
+logfire.configure()
+logfire.instrument_pydantic_ai()
+
+# Make a synchronous request to the model
+model_response = model_request_sync(
+    'anthropic:claude-haiku-4-5',
+    [ModelRequest.user_text_prompt('What is the capital of France?')],
+)
+
+print(model_response.parts[0].content)
+#> The capital of France is Paris.
+```
+
+_(This example is complete, it can be run "as is")_
+
+You can also enable OpenTelemetry on a per call basis:
+
+```python {title="direct_instrumented.py" hl_lines="1 6 12"}
+import logfire
+
+from pydantic_ai import ModelRequest
+from pydantic_ai.direct import model_request_sync
+
+logfire.configure()
+
+# Make a synchronous request to the model
+model_response = model_request_sync(
+    'anthropic:claude-haiku-4-5',
+    [ModelRequest.user_text_prompt('What is the capital of France?')],
+    instrument=True
+)
+
+print(model_response.parts[0].content)
+#> The capital of France is Paris.
+```
+
+See [Debugging and Monitoring](logfire.md) for more details, including how to instrument with plain OpenTelemetry without Logfire.

docs/durable_execution/airflow.md

@@ -0,0 +1,113 @@
+# Durable Execution with Apache Airflow
+
+[Apache Airflow](https://airflow.apache.org) is a workflow orchestrator. Its Pydantic AI integration is provided by the [`apache-airflow-providers-common-ai`](https://airflow.apache.org/docs/apache-airflow-providers-common-ai/stable/index.html) package through `airflow.providers.common.ai`, rather than by `pydantic_ai.durable_exec`.
+
+Unlike the wrapper-object integrations on this page, Airflow's durable unit is an **Airflow task**. You author a normal Pydantic AI agent and run it as a task; Airflow's retry machinery plus a step-level cache resume the agent from its last completed model request or tool call instead of replaying the whole run.
+
+## Durable Execution
+
+When an agent runs as a durable Airflow task, Airflow records each completed **model request** and **tool call** as a cache entry. On a retry, Airflow replays these entries to skip completed work: each entry stores a fingerprint of the request that produced it, and if that fingerprint no longer matches (the conversation diverged since the previous attempt), the step re-runs live instead of returning a stale result. The cache lives in object storage (local, S3, GCS, or Azure) for the lifetime of a single DAG run's task and is deleted when the task succeeds.
+
+For example, imagine an agent calls a model, gets a useful response, starts a tool call, and then the worker crashes. Without durable execution, Airflow's normal retry restarts the task from the top and repeats the model request and any later side effects. With durable execution, the retry replays the run, reuses the cached result for the already-completed model request, and continues from the first operation that has not completed.
+
+This is useful for long-running agents and for runs where a repeated model request or external tool call would cost money, take time, or duplicate a side effect.
+
+## Durable Agent
+
+You make an agent durable by running it through Airflow's `AgentOperator` (or the `@task.agent` decorator) with `durable=True`. Install the provider alongside Airflow:
+
+```bash
+uv add "apache-airflow-providers-common-ai"
+```
+
+The agent's model and credentials come from an Airflow connection (the examples use `pydanticai_default`). See [Pydantic AI connection](https://airflow.apache.org/docs/apache-airflow-providers-common-ai/stable/connections/pydantic_ai.html) for how to configure one.
+
+Durable execution needs a place to store its step cache. Point `[common.ai] durable_cache_path` at an object-storage location:
+
+```ini title="airflow.cfg"
+[common.ai]
+durable_cache_path = s3://my-bucket/airflow-agent-cache
+```
+
+Here is the smallest durable Pydantic AI agent as an Airflow task:
+
+```python {title="durable_agent_dag.py" test="skip" lint="skip"}
+from datetime import timedelta
+
+from airflow.providers.common.ai.operators.agent import AgentOperator
+from airflow.sdk import dag
+
+
+@dag(default_args={"retries": 3, "retry_delay": timedelta(seconds=30)})
+def durable_agent_dag():
+    AgentOperator(
+        task_id="researcher",
+        prompt="Summarize quantum error correction.",
+        llm_conn_id="pydanticai_default",
+        durable=True,
+    )
+
+
+durable_agent_dag()
+```
+
+`AgentOperator` does not replace the underlying Pydantic AI agent. It builds the agent from your connection and toolsets, then wraps the model and toolsets so that, while the task runs, it records recoverable operations:
+
+* model requests;
+* Pydantic AI tool calls.
+
+The same applies to the `@task.agent` decorator, where the decorated function returns the prompt:
+
+```python {title="durable_agent_decorator.py" test="skip" lint="skip"}
+from datetime import timedelta
+
+from airflow.providers.common.ai.toolsets.sql import SQLToolset
+from airflow.sdk import dag, task
+
+
+@dag(default_args={"retries": 3, "retry_delay": timedelta(seconds=30)})
+def durable_agent_decorator():
+    @task.agent(
+        llm_conn_id="pydanticai_default",
+        system_prompt="You are a data analyst. Use tools to answer questions.",
+        durable=True,
+        toolsets=[SQLToolset(db_conn_id="postgres_default", allowed_tables=["orders"])],
+    )
+    def analyze(question: str) -> str:
+        return f"Answer this question about our orders data: {question}"
+
+    analyze("What was our total revenue last month?")
+
+
+durable_agent_decorator()
+```
+
+The agent's retries are Airflow task retries: configure them with the task's `retries` and `retry_delay`. On each retry the cached steps are replayed and the run continues from the first operation that has not completed. For the full reference, see the Airflow [`AgentOperator` durable execution docs](https://airflow.apache.org/docs/apache-airflow-providers-common-ai/stable/operators/agent.html#durable-execution).
+
+## Tools and side effects
+
+Durable execution caches the result of each Pydantic AI tool call, including tools backed by Airflow toolsets (`SQLToolset`, `HookToolset`, `MCPToolset`, and others). On replay the cached result is returned without re-invoking the tool, so a tool that writes to an external system runs at most once per completed step across all retries of a run.
+
+## Human-in-the-loop
+
+Airflow's `AgentOperator` has a separate human-in-the-loop review mode (`enable_hitl_review=True`) that pauses an agent run for human approval, rejection, or change requests through Airflow's HITL UI.
+
+!!! warning
+
+    `durable=True` and `enable_hitl_review=True` cannot be combined today. A durable run replays deterministically from its cache and does not pause for human input; a human-in-the-loop run is interactive and is not yet captured by the step cache. Choose one per task.
+
+## Streaming
+
+Streaming is not yet supported under `durable=True`. The durable model wrapper records complete model requests, not streamed events. Run streaming agents as non-durable tasks.
+
+## Requirements and Constraints
+
+When running a Pydantic AI agent as a durable Airflow task:
+
+* The durable unit is the Airflow task; recovery happens through Airflow task retries, so set `retries` (and a `retry_delay`) on the task.
+* Define the agent with a concrete model, for example via a connection that resolves to `Agent('openai:gpt-5-nano', ...)`. The model must be set when `durable=True`.
+* Set `[common.ai] durable_cache_path` to an object-storage location the workers can read and write.
+* On a retry, a cached model request or tool call is replayed when its stored fingerprint matches the current request, and re-runs live when it diverges. Requests that can't be serialized to a fingerprint fall back to unverified positional replay, so keep runs deterministic across retries.
+* `durable=True` and `enable_hitl_review=True` are mutually exclusive.
+* Streaming is not supported under `durable=True`.
+* The step cache is scoped to one DAG run's task and is deleted when the task succeeds.

docs/durable_execution/dbos.md

@@ -0,0 +1,172 @@
+# Durable Execution with DBOS
+
+[DBOS](https://www.dbos.dev/) is a lightweight [durable execution](https://docs.dbos.dev/architecture) library natively integrated with Pydantic AI.
+
+## Durable Execution
+
+DBOS workflows make your program **durable** by checkpointing its state in a database. If your program ever fails, when it restarts all your workflows will automatically resume from the last completed step.
+
+* **Workflows** must be deterministic and generally cannot include I/O.
+* **Steps** may perform I/O (network, disk, API calls). If a step fails, it restarts from the beginning.
+
+Every workflow input and step output is durably stored in the system database. When workflow execution fails, whether from crashes, network issues, or server restarts, DBOS leverages these checkpoints to recover workflows from their last completed step.
+
+DBOS **queues** provide durable, database-backed alternatives to systems like Celery or BullMQ, supporting features such as concurrency limits, rate limits, timeouts, and prioritization. See the [DBOS docs](https://docs.dbos.dev/architecture) for details.
+
+The diagram below shows the overall architecture of an agentic application in DBOS.
+DBOS runs fully in-process as a library. Functions remain normal Python functions but are checkpointed into a database (Postgres or SQLite).
+
+```text
+                    Clients
+            (HTTP, RPC, Kafka, etc.)
+                        |
+                        v
++------------------------------------------------------+
+|               Application Servers                    |
+|                                                      |
+|   +----------------------------------------------+   |
+|   |        Pydantic AI + DBOS Libraries          |   |
+|   |                                              |   |
+|   |  [ Workflows (Agent Run Loop) ]              |   |
+|   |  [ Steps (Tool, MCP, Model) ]                |   |
+|   |  [ Queues ]   [ Cron Jobs ]   [ Messaging ]  |   |
+|   +----------------------------------------------+   |
+|                                                      |
++------------------------------------------------------+
+                        |
+                        v
++------------------------------------------------------+
+|                      Database                        |
+|   (Stores workflow and step state, schedules tasks)  |
++------------------------------------------------------+
+```
+
+See the [DBOS documentation](https://docs.dbos.dev/architecture) for more information.
+
+## Durable Agent
+
+Any agent can be wrapped in a [`DBOSAgent`][pydantic_ai.durable_exec.dbos.DBOSAgent] to get durable execution. `DBOSAgent` automatically:,
+
+* Wraps `Agent.run` and `Agent.run_sync` as DBOS workflows.
+* Wraps [model requests](../models/overview.md) and [MCP communication](../mcp/client.md) as DBOS steps.
+
+Custom tool functions and event stream handlers are **not automatically wrapped** by DBOS.
+If they involve non-deterministic behavior or perform I/O, you should explicitly decorate them with `@DBOS.step`.
+
+The original agent, model, and MCP server can still be used as normal outside the DBOS workflow.
+
+Here is a simple but complete example of wrapping an agent for durable execution. All it requires is to install Pydantic AI with the DBOS [open-source library](https://github.com/dbos-inc/dbos-transact-py):
+
+```bash
+pip/uv-add pydantic-ai[dbos]
+```
+
+Or if you're using the slim package, you can install it with the `dbos` optional group:
+
+```bash
+pip/uv-add pydantic-ai-slim[dbos]
+```
+
+```python {title="dbos_agent.py" test="skip"}
+from dbos import DBOS, DBOSConfig
+
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.dbos import DBOSAgent
+
+dbos_config: DBOSConfig = {
+    'name': 'pydantic_dbos_agent',
+    'system_database_url': 'sqlite:///dbostest.sqlite',  # (3)!
+}
+DBOS(config=dbos_config)
+
+agent = Agent(
+    'gpt-5.2',
+    instructions="You're an expert in geography.",
+    name='geography',  # (4)!
+)
+
+dbos_agent = DBOSAgent(agent)  # (1)!
+
+async def main():
+    DBOS.launch()
+    result = await dbos_agent.run('What is the capital of Mexico?')  # (2)!
+    print(result.output)
+    #> Mexico City (Ciudad de México, CDMX)
+```
+
+1. Workflows and `DBOSAgent` must be defined before `DBOS.launch()` so that recovery can correctly find all workflows.
+2. [`DBOSAgent.run()`][pydantic_ai.durable_exec.dbos.DBOSAgent.run] works like [`Agent.run()`][pydantic_ai.agent.Agent.run], but runs as a DBOS workflow and executes model requests, decorated tool calls, and MCP communication as DBOS steps.
+3. This example uses SQLite. Postgres is recommended for production.
+4. The agent's `name` is used to uniquely identify its workflows.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+Because DBOS workflows need to be defined before calling `DBOS.launch()` and the `DBOSAgent` instance automatically registers `run` and `run_sync` as workflows, it needs to be defined before calling `DBOS.launch()` as well.
+
+For more information on how to use DBOS in Python applications, see their [Python SDK guide](https://docs.dbos.dev/python/programming-guide).
+
+## DBOS Integration Considerations
+
+When using DBOS with Pydantic AI agents, there are a few important considerations to ensure workflows and toolsets behave correctly.
+
+### Agent and Toolset Requirements
+
+Each agent instance must have a unique `name` so DBOS can correctly resume workflows after a failure or restart.
+
+Each [`MCPToolset`][pydantic_ai.mcp.MCPToolset] must have a unique [`id`][pydantic_ai.toolsets.AbstractToolset.id], as DBOS derives its step names and per-run tool-defs cache key from it. This field is normally optional, but is required when using DBOS. It should not be changed once the durable agent has been deployed to production, as this would break active workflows.
+
+Tools and event stream handlers are not automatically wrapped by DBOS. You can decide how to integrate them:
+
+* Decorate with `@DBOS.step` if the function involves non-determinism or I/O.
+* Skip the decorator if durability isn't needed, so you avoid the extra DB checkpoint write.
+* If the function needs to enqueue tasks or invoke other DBOS workflows, run it inside the agent's main workflow (not as a step).
+
+Other than that, any agent and toolset will just work!
+
+### Agent Run Context and Dependencies
+
+DBOS checkpoints workflow inputs/outputs and step outputs into a database using [`pickle`](https://docs.python.org/3/library/pickle.html). This means you need to make sure [dependencies](../dependencies.md) object provided to [`DBOSAgent.run()`][pydantic_ai.durable_exec.dbos.DBOSAgent.run] or [`DBOSAgent.run_sync()`][pydantic_ai.durable_exec.dbos.DBOSAgent.run_sync], and tool outputs can be serialized using pickle. You may also want to keep the inputs and outputs small (under \~2 MB). PostgreSQL and SQLite support up to 1 GB per field, but large objects may impact performance.
+
+### Streaming
+
+Because DBOS cannot stream output directly to the workflow or step call site, [`Agent.run_stream()`][pydantic_ai.agent.Agent.run_stream] and [`Agent.run_stream_events()`][pydantic_ai.agent.Agent.run_stream_events] are not supported when running inside of a DBOS workflow.
+
+Instead, you can implement streaming by setting an [`event_stream_handler`][pydantic_ai.agent.EventStreamHandler] on the `Agent` or `DBOSAgent` instance and using [`DBOSAgent.run()`][pydantic_ai.durable_exec.dbos.DBOSAgent.run].
+The event stream handler function will receive the agent [run context][pydantic_ai.tools.RunContext] and an async iterable of events from the model's streaming response and the agent's execution of tools. For examples, see the [streaming docs](../agent.md#streaming-all-events).
+
+
+### Parallel Tool Execution
+
+When using `DBOSAgent`, tools are executed in parallel by default to minimize latency. To guarantee deterministic replay and reliable recovery, DBOS waits for all parallel tool calls to complete before emitting events **in order**.
+It's equivalent to the behavior of [`with agent.parallel_tool_call_execution_mode('parallel_ordered_events')`][pydantic_ai.agent.AbstractAgent.parallel_tool_call_execution_mode].
+
+If you prefer strict ordering, you can configure the agent to run tools sequentially by setting [`parallel_execution_mode='sequential'`][pydantic_ai.durable_exec.dbos.DBOSAgent] when initializing the `DBOSAgent`.
+
+### Toolsets at Runtime
+
+Additional toolsets can be passed per run via [`DBOSAgent.run(toolsets=...)`][pydantic_ai.durable_exec.dbos.DBOSAgent.run]. Non-executing toolsets like [`ExternalToolset`][pydantic_ai.toolsets.ExternalToolset], and [`FunctionToolset`][pydantic_ai.toolsets.FunctionToolset]s whose tools DBOS runs inline, are supported. [`MCPToolset`][pydantic_ai.mcp.MCPToolset]s and dynamic toolsets must be set when constructing the agent so their steps are registered before the workflow runs; passing them at runtime raises a `UserError`.
+
+
+## Step Configuration
+
+You can customize DBOS step behavior, such as retries, by passing [`StepConfig`][pydantic_ai.durable_exec.dbos.StepConfig] objects to the `DBOSAgent` constructor:
+
+- `mcp_step_config`: The DBOS step config to use for MCP server communication. No retries if omitted.
+- `model_step_config`: The DBOS step config to use for model request steps. No retries if omitted.
+
+For custom tools, you can annotate them directly with [`@DBOS.step`](https://docs.dbos.dev/python/reference/decorators#step) or [`@DBOS.workflow`](https://docs.dbos.dev/python/reference/decorators#workflow) decorators as needed. These decorators have no effect outside DBOS workflows, so tools remain usable in non-DBOS agents.
+
+
+## Step Retries
+
+On top of the automatic retries for request failures that DBOS will perform, Pydantic AI and various provider API clients also have their own request retry logic. Enabling these at the same time may cause the request to be retried more often than expected, with improper `Retry-After` handling.
+
+When using DBOS, it's recommended to not use [HTTP Request Retries](../retries.md) and to turn off your provider API client's own retry logic, for example by setting `max_retries=0` on a [custom `OpenAIProvider` API client](../models/openai.md#custom-openai-client).
+
+You can customize DBOS's retry policy using [step configuration](#step-configuration).
+
+## Observability with Logfire
+
+DBOS can be configured to generate OpenTelemetry spans for each workflow and step execution, and Pydantic AI emits spans for each agent run, model request, and tool invocation. You can send these spans to [Pydantic Logfire](../logfire.md) to get a full, end-to-end view of what's happening in your application.
+
+For more information about DBOS logging and tracing, please see the [DBOS docs](https://docs.dbos.dev/python/tutorials/logging-and-tracing) for details.

docs/durable_execution/kitaru.md

@@ -0,0 +1,127 @@
+# Durable Execution with Kitaru
+
+[Kitaru](https://docs.zenml.io/kitaru) is a durable execution layer for AI agents. Its Pydantic AI adapter is provided by the `kitaru` package through `kitaru.adapters.pydantic_ai`, rather than by `pydantic_ai.durable_exec`.
+
+## Durable Execution
+
+Kitaru records agent progress as **flows** and **checkpoints**. A flow is the durable run you can resume later. A checkpoint is a completed model request, tool call, MCP invocation, or human wait that Kitaru can reuse during recovery.
+
+For example, imagine an agent calls a model, gets a useful response, starts a tool call, and then the process crashes. Without durable execution, restarting the program usually repeats the model request and may repeat later side effects too. With Kitaru, the restarted flow can replay the run, reuse the completed checkpoint for the model request, and continue from the first incomplete point.
+
+This is useful for long-running agents, human-in-the-loop workflows, and applications where a repeated model request or external API call would cost money, take time, or duplicate a side effect.
+
+When you call a `KitaruAgent`, your application still calls the underlying Pydantic AI agent. Kitaru starts or resumes a flow for that call. With the default `"calls"` checkpoint strategy, it records completed model requests, tool calls, MCP invocations, and human waits in Kitaru's checkpoint storage. On recovery, Kitaru runs the Python function again until it reaches an operation that already has a checkpoint, returns the saved result for that operation, and then continues from the first operation that has not completed.
+
+## Durable Agent
+
+You can make a normal [`Agent`][pydantic_ai.agent.Agent] durable by wrapping it with `KitaruAgent` from `kitaru.adapters.pydantic_ai`. Install Kitaru separately from Pydantic AI:
+
+```bash
+uv add "kitaru[pydantic-ai]"
+```
+
+For local development with Kitaru's local server, install the `local` extra too:
+
+```bash
+uv add "kitaru[pydantic-ai,local]"
+```
+
+Initialize the project and check your connection before running the examples:
+
+```bash
+kitaru init
+kitaru login
+kitaru status
+```
+
+Here is the smallest durable Pydantic AI agent using Kitaru:
+
+```python {title="kitaru_agent.py" test="skip" lint="skip"}
+from pydantic_ai import Agent
+from kitaru.adapters.pydantic_ai import KitaruAgent
+
+agent = Agent('openai:gpt-5-nano', name='researcher')
+durable_agent = KitaruAgent(agent)
+
+result = durable_agent.run_sync('Summarize quantum error correction.')
+print(result.output)
+```
+
+`KitaruAgent` does not replace the original agent object. With the default call-level checkpoint strategy, it delegates to that agent for the actual Pydantic AI run and records recoverable operations while the run is executing:
+
+* model requests;
+* Pydantic AI tool calls;
+* MCP tool calls;
+* `@hitl_tool` human waits.
+
+It exposes the usual run methods, including [`Agent.run`][pydantic_ai.agent.Agent.run] and [`Agent.run_sync`][pydantic_ai.agent.Agent.run_sync]. The original [`Agent`][pydantic_ai.agent.Agent] can still be used normally outside Kitaru.
+
+## Production Flows
+
+!!! warning
+
+    The minimal wrapper above uses Kitaru's automatic flow creation, which is intended for local development. For remote stacks or production services, put the durable agent call inside an explicit `@kitaru.flow` so Kitaru has a stable flow entry point to submit, replay, and inspect.
+
+```python {title="kitaru_flow.py" test="skip" lint="skip"}
+import kitaru
+from pydantic_ai import Agent
+from kitaru.adapters.pydantic_ai import KitaruAgent
+
+agent = Agent('openai:gpt-5-nano', name='researcher')
+durable_agent = KitaruAgent(agent)
+
+
+@kitaru.flow
+def research_topic(topic: str) -> str:
+    result = durable_agent.run_sync(f'Summarize {topic}.')
+    return result.output
+```
+
+Use the short wrapper for first experiments. Use an explicit flow when the run needs to move beyond your local process.
+
+## Checkpoint Strategy
+
+`KitaruAgent` supports two checkpoint strategies:
+
+| Strategy | Default? | What gets persisted | Best for |
+| --- | --- | --- | --- |
+| `"calls"` | Yes | Replay-safe model requests, tool calls, MCP invocations, and human waits are persisted as separate checkpoints. | Most agents, especially when individual calls are expensive or have side effects. |
+| `"turn"` | No | One checkpoint wraps the full agent run. | Simpler runs where per-call checkpoints are unnecessary, or cases where streaming constraints require a full-turn checkpoint. |
+
+With the default `"calls"` strategy, Kitaru cannot create nested checkpoints inside a user-defined `@kitaru.checkpoint` body. If `durable_agent.run_sync(...)` runs inside a user `@kitaru.checkpoint`, Kitaru records the whole agent turn under that outer checkpoint instead of creating separate model, tool, or MCP checkpoint rows.
+
+See the [Kitaru Pydantic AI adapter guide](https://docs.zenml.io/kitaru/adapters/pydantic-ai) for advanced checkpoint configuration.
+
+## Human-in-the-loop
+
+For pure human approval or data-entry gates, prefer Kitaru's `@hitl_tool`. It turns the wait into a durable tool call: the process can stop while waiting for a human response, and recovery can continue after the response is available.
+
+```python {test="skip" lint="skip"}
+from kitaru.adapters.pydantic_ai import hitl_tool
+
+
+@hitl_tool(question='Approve publishing this answer?', schema=bool)
+def approve_publish(summary: str) -> bool: ...
+```
+
+!!! warning "Regular tool-body waits need extra configuration"
+
+    Regular Pydantic AI tool bodies can also wait for human input, but sync tool-body waits need extra Kitaru configuration. If you need that pattern, follow the human-in-the-loop section of the [Kitaru Pydantic AI adapter guide](https://docs.zenml.io/kitaru/adapters/pydantic-ai#human-in-the-loop).
+
+For Pydantic AI's own deferred tool patterns, see [deferred tools](../deferred-tools.md).
+
+## Streaming
+
+Kitaru supports Pydantic AI streaming with some constraints. For event streaming, prefer the [`event_stream_handler`](../agent.md#streaming-all-events) argument on [`Agent.run`][pydantic_ai.agent.Agent.run]. When a run uses `event_stream_handler`, Kitaru falls back to a turn checkpoint for that call.
+
+If you use [`run_stream()`][pydantic_ai.agent.AbstractAgent.run_stream] or [`iter()`][pydantic_ai.agent.AbstractAgent.iter], wrap the streaming call in an explicit `@kitaru.checkpoint`. This gives Kitaru one durable operation to replay instead of trying to persist each streamed event separately.
+
+## Requirements and Constraints
+
+When using Kitaru with Pydantic AI:
+
+* Define the agent with a concrete model at construction time, such as `Agent('openai:gpt-5-nano', ...)`.
+* Give each durable agent a stable `name`; Kitaru uses it to identify persisted work across runs.
+* Do not override the model per run with `model=` when using `KitaruAgent`.
+* Use an explicit `@kitaru.flow` for remote stacks and production services; automatic flow creation is local-only.
+* Avoid nested Kitaru checkpoints inside user-defined `@kitaru.checkpoint` bodies.

docs/durable_execution/overview.md

@@ -0,0 +1,17 @@
+# Durable Execution
+
+Pydantic AI allows you to build durable agents that can preserve their progress across transient API failures and application errors or restarts, and handle long-running, asynchronous, and human-in-the-loop workflows with production-grade reliability. Durable agents have full support for [streaming](../agent.md#streaming-all-events) and [MCP](../mcp/client.md), with the added benefit of fault tolerance.
+
+Pydantic AI officially supports four durable execution solutions:
+
+- [Temporal](./temporal.md)
+- [DBOS](./dbos.md)
+- [Prefect](./prefect.md)
+- [Restate](./restate.md)
+
+These integrations are co-maintained by the Pydantic and vendor teams and only use Pydantic AI's public interface, so they also serve as a reference for integrating with other durable systems.
+
+Additional external SDK integrations:
+
+- [Kitaru](./kitaru.md)
+- [Apache Airflow](./airflow.md)

docs/durable_execution/prefect.md

@@ -0,0 +1,297 @@
+# Durable Execution with Prefect
+
+[Prefect](https://www.prefect.io/) is a workflow orchestration framework for building resilient data pipelines in Python, natively integrated with Pydantic AI.
+
+## Durable Execution
+
+Prefect 3.0 brings [transactional semantics](https://www.prefect.io/blog/transactional-ml-pipelines-with-prefect-3-0) to your Python workflows, allowing you to group tasks into atomic units and define failure modes. If any part of a transaction fails, the entire transaction can be rolled back to a clean state.
+
+* **Flows** are the top-level entry points for your workflow. They can contain tasks and other flows.
+* **Tasks** are individual units of work that can be retried, cached, and monitored independently.
+
+Prefect 3.0's approach to transactional orchestration makes your workflows automatically **idempotent**: rerunnable without duplication or inconsistency across any environment. Every task is executed within a transaction that governs when and where the task's result record is persisted. If the task runs again under an identical context, it will not re-execute but instead load its previous result.
+
+The diagram below shows the overall architecture of an agentic application with Prefect.
+Prefect uses client-side task orchestration by default, with optional server connectivity for advanced features like scheduling and monitoring.
+
+```text
+            +---------------------+
+            |   Prefect Server    |      (Monitoring,
+            |      or Cloud       |       scheduling, UI,
+            +---------------------+       orchestration)
+                     ^
+                     |
+        Flow state,  |   Schedule flows,
+        metadata,    |   track execution
+        logs         |
+                     |
++------------------------------------------------------+
+|               Application Process                    |
+|   +----------------------------------------------+   |
+|   |              Flow (Agent.run)                |   |
+|   +----------------------------------------------+   |
+|          |          |                |               |
+|          v          v                v               |
+|   +-----------+ +------------+ +-------------+       |
+|   |   Task    | |    Task    | |    Task     |       |
+|   |  (Tool)   | | (MCP Tool) | | (Model API) |       |
+|   +-----------+ +------------+ +-------------+       |
+|         |           |                |               |
+|       Cache &     Cache &          Cache &           |
+|       persist     persist          persist           |
+|         to           to               to             |
+|         v            v                v              |
+|   +----------------------------------------------+   |
+|   |     Result Storage (Local FS, S3, etc.)     |    |
+|   +----------------------------------------------+   |
++------------------------------------------------------+
+          |           |                |
+          v           v                v
+      [External APIs, services, databases, etc.]
+```
+
+See the [Prefect documentation](https://docs.prefect.io/) for more information.
+
+## Durable Agent
+
+Any agent can be wrapped in a [`PrefectAgent`][pydantic_ai.durable_exec.prefect.PrefectAgent] to get durable execution. `PrefectAgent` automatically:
+
+* Wraps [`Agent.run`][pydantic_ai.agent.Agent.run] and [`Agent.run_sync`][pydantic_ai.agent.Agent.run_sync] as Prefect flows.
+* Wraps [model requests](../models/overview.md) as Prefect tasks.
+* Wraps [tool calls](../tools.md) as Prefect tasks (configurable per-tool).
+* Wraps [MCP communication](../mcp/client.md) as Prefect tasks.
+
+Event stream handlers are **automatically wrapped** by Prefect when running inside a Prefect flow. Each event from the stream is processed in a separate Prefect task for durability. You can customize the task behavior using the `event_stream_handler_task_config` parameter when creating the `PrefectAgent`. Do **not** manually decorate event stream handlers with `@task`. For examples, see the [streaming docs](../agent.md#streaming-all-events)
+
+The original agent, model, and MCP server can still be used as normal outside the Prefect flow.
+
+Here is a simple but complete example of wrapping an agent for durable execution. All it requires is to install Pydantic AI with Prefect:
+
+```bash
+pip/uv-add pydantic-ai[prefect]
+```
+
+Or if you're using the slim package, you can install it with the `prefect` optional group:
+
+```bash
+pip/uv-add pydantic-ai-slim[prefect]
+```
+
+```python {title="prefect_agent.py" test="skip"}
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.prefect import PrefectAgent
+
+agent = Agent(
+    'gpt-5.2',
+    instructions="You're an expert in geography.",
+    name='geography',  # (1)!
+)
+
+prefect_agent = PrefectAgent(agent)  # (2)!
+
+async def main():
+    result = await prefect_agent.run('What is the capital of Mexico?')  # (3)!
+    print(result.output)
+    #> Mexico City (Ciudad de México, CDMX)
+```
+
+1. The agent's `name` is used to uniquely identify its flows and tasks.
+2. Wrapping the agent with `PrefectAgent` enables durable execution for all agent runs.
+3. [`PrefectAgent.run()`][pydantic_ai.durable_exec.prefect.PrefectAgent.run] works like [`Agent.run()`][pydantic_ai.agent.Agent.run], but runs as a Prefect flow and executes model requests, decorated tool calls, and MCP communication as Prefect tasks.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+For more information on how to use Prefect in Python applications, see their [Python documentation](https://docs.prefect.io/v3/how-to-guides/workflows/write-and-run).
+
+## Prefect Integration Considerations
+
+When using Prefect with Pydantic AI agents, there are a few important considerations to ensure workflows behave correctly.
+
+### Agent Requirements
+
+Each agent instance must have a unique `name` so Prefect can correctly identify and track its flows and tasks.
+
+### Tool Wrapping
+
+Agent tools are automatically wrapped as Prefect tasks, which means they benefit from:
+
+* **Retry logic**: Failed tool calls can be retried automatically
+* **Caching**: Tool results are cached based on their inputs
+* **Observability**: Tool execution is tracked in the Prefect UI
+
+You can customize tool task behavior using `tool_task_config` (applies to all tools) or `tool_task_config_by_name` (per-tool configuration):
+
+```python {title="prefect_agent_config.py" test="skip"}
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.prefect import PrefectAgent, TaskConfig
+
+agent = Agent('openai:gpt-5.2', name='my_agent')
+
+@agent.tool_plain
+def fetch_data(url: str) -> str:
+    # This tool will be wrapped as a Prefect task
+    ...
+
+prefect_agent = PrefectAgent(
+    agent,
+    tool_task_config=TaskConfig(retries=3),  # Default for all tools
+    tool_task_config_by_name={
+        'fetch_data': TaskConfig(timeout_seconds=10.0),  # Specific to fetch_data
+        'simple_tool': None,  # Disable task wrapping for simple_tool
+    },
+)
+```
+
+Set a tool's config to `None` in `tool_task_config_by_name` to disable task wrapping for that specific tool.
+
+### Streaming
+
+When running inside a Prefect flow, [`Agent.run_stream()`][pydantic_ai.agent.Agent.run_stream] works but doesn't provide real-time streaming because Prefect tasks consume their entire execution before returning results. The method will execute fully and return the complete result at once.
+
+For real-time streaming behavior inside Prefect flows, you can set an [`event_stream_handler`][pydantic_ai.agent.EventStreamHandler] on the `Agent` or `PrefectAgent` instance and use [`PrefectAgent.run()`][pydantic_ai.durable_exec.prefect.PrefectAgent.run].
+
+**Note**: Event stream handlers behave differently when running inside a Prefect flow versus outside:
+- **Outside a flow**: The handler receives events as they stream from the model
+- **Inside a flow**: Each event is wrapped as a Prefect task for durability, which may affect timing but ensures reliability
+
+The event stream handler function will receive the agent [run context][pydantic_ai.tools.RunContext] and an async iterable of events from the model's streaming response and the agent's execution of tools. For examples, see the [streaming docs](../agent.md#streaming-all-events).
+
+### Toolsets at Runtime
+
+Additional toolsets can be passed per run via [`PrefectAgent.run(toolsets=...)`][pydantic_ai.durable_exec.prefect.PrefectAgent.run], but only non-executing toolsets like [`ExternalToolset`][pydantic_ai.toolsets.ExternalToolset], whose tools are executed outside the agent run, are supported. Executing toolsets ([`FunctionToolset`][pydantic_ai.toolsets.FunctionToolset] and [`MCPToolset`][pydantic_ai.mcp.MCPToolset]) and dynamic toolsets must be set when constructing the agent so their tasks are registered before the flow runs; passing them at runtime raises a `UserError`.
+
+## Task Configuration
+
+You can customize Prefect task behavior, such as retries and timeouts, by passing [`TaskConfig`][pydantic_ai.durable_exec.prefect.TaskConfig] objects to the `PrefectAgent` constructor:
+
+- `mcp_task_config`: Configuration for MCP server communication tasks
+- `model_task_config`: Configuration for model request tasks
+- `tool_task_config`: Default configuration for all tool calls
+- `tool_task_config_by_name`: Per-tool task configuration (overrides `tool_task_config`)
+- `event_stream_handler_task_config`: Configuration for event stream handler tasks (applies when running inside a Prefect flow)
+
+Available `TaskConfig` options:
+
+- `retries`: Maximum number of retries for the task (default: `0`)
+- `retry_delay_seconds`: Delay between retries in seconds (can be a single value or list for exponential backoff, default: `1.0`)
+- `timeout_seconds`: Maximum time in seconds for the task to complete
+- `cache_policy`: Custom Prefect cache policy for the task
+- `persist_result`: Whether to persist the task result
+- `result_storage`: Prefect result storage for the task (e.g., `'s3-bucket/my-storage'` or a `WritableFileSystem` block)
+- `log_prints`: Whether to log print statements from the task (default: `False`)
+
+Example:
+
+```python {title="prefect_agent_config.py" test="skip"}
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.prefect import PrefectAgent, TaskConfig
+
+agent = Agent(
+    'gpt-5.2',
+    instructions="You're an expert in geography.",
+    name='geography',
+)
+
+prefect_agent = PrefectAgent(
+    agent,
+    model_task_config=TaskConfig(
+        retries=3,
+        retry_delay_seconds=[1.0, 2.0, 4.0],  # Exponential backoff
+        timeout_seconds=30.0,
+    ),
+)
+
+async def main():
+    result = await prefect_agent.run('What is the capital of France?')
+    print(result.output)
+    #> Paris
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+### Retry Considerations
+
+Pydantic AI and provider API clients have their own retry logic. When using Prefect, you may want to:
+
+* Disable [HTTP Request Retries](../retries.md) in Pydantic AI
+* Turn off your provider API client's retry logic (e.g., `max_retries=0` on a [custom OpenAI client](../models/openai.md#custom-openai-client))
+* Rely on Prefect's task-level retry configuration for consistency
+
+This prevents requests from being retried multiple times at different layers.
+
+## Caching and Idempotency
+
+Prefect 3.0 provides built-in caching and transactional semantics. Tasks with identical inputs will not re-execute if their results are already cached, making workflows naturally idempotent and resilient to failures.
+
+* **Task inputs**: Messages, settings, parameters, tool arguments, and serializable dependencies
+
+**Note**: For user dependencies to be included in cache keys, they must be serializable (e.g., Pydantic models or basic Python types). Non-serializable dependencies are automatically excluded from cache computation.
+
+## Observability with Prefect and Logfire
+
+Prefect provides a built-in UI for monitoring flow runs, task executions, and failures. You can:
+
+* View real-time flow run status
+* Debug failures with full stack traces
+* Set up alerts and notifications
+
+To access the Prefect UI, you can either:
+
+1. Use [Prefect Cloud](https://www.prefect.io/cloud) (managed service)
+2. Run a local [Prefect server](https://docs.prefect.io/v3/how-to-guides/self-hosted/server-cli) with `prefect server start`
+
+You can also use [Pydantic Logfire](../logfire.md) for detailed observability. When using both Prefect and Logfire, you'll get complementary views:
+
+* **Prefect**: Workflow-level orchestration, task status, and retry history
+* **Logfire**: Fine-grained tracing of agent runs, model requests, and tool invocations
+
+When using Logfire with Prefect, you can enable distributed tracing to see spans for your Prefect runs included with your agent runs, model requests, and tool invocations.
+
+For more information about Prefect monitoring, see the [Prefect documentation](https://docs.prefect.io/).
+
+## Deployments and Scheduling
+
+To deploy and schedule a `PrefectAgent`, wrap it in a Prefect flow and use the flow's [`serve()`](https://docs.prefect.io/v3/how-to-guides/deployments/create-deployments#create-a-deployment-with-serve) or [`deploy()`](https://docs.prefect.io/v3/how-to-guides/deployments/deploy-via-python) methods:
+
+```python {title="serve_agent.py" test="skip"}
+from prefect import flow
+
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.prefect import PrefectAgent
+
+
+@flow
+async def daily_report_flow(user_prompt: str):
+    """Generate a daily report using the agent."""
+    agent = Agent(  # (1)!
+        'openai:gpt-5.2',
+        name='daily_report_agent',
+        instructions='Generate a daily summary report.',
+    )
+
+    prefect_agent = PrefectAgent(agent)
+
+    result = await prefect_agent.run(user_prompt)
+    return result.output
+
+
+
+# Serve the flow with a daily schedule
+if __name__ == '__main__':
+    daily_report_flow.serve(
+        name='daily-report-deployment',
+        cron='0 9 * * *',  # Run daily at 9am
+        parameters={'user_prompt': "Generate today's report"},
+        tags=['production', 'reports'],
+    )
+```
+
+1. Each flow run executes in an isolated process, and all inputs and dependencies must be serializable. Because Agent instances cannot be serialized, instantiate the agent inside the flow rather than at the module level.
+
+The `serve()` method accepts scheduling options:
+
+- **`cron`**: Cron schedule string (e.g., `'0 9 * * *'` for daily at 9am)
+- **`interval`**: Schedule interval in seconds or as a timedelta
+- **`rrule`**: iCalendar RRule schedule string
+
+For production deployments with Docker, Kubernetes, or other infrastructure, use the flow's [`deploy()`](https://docs.prefect.io/v3/how-to-guides/deployments/deploy-via-python) method. See the [Prefect deployment documentation](https://docs.prefect.io/v3/how-to-guides/deployments/create-deploymentsy) for more information.

docs/durable_execution/restate.md

@@ -0,0 +1,113 @@
+# Durable Execution with Restate
+
+[Restate](https://restate.dev) is a lightweight durable execution runtime with first-class support for AI agents. The Pydantic AI integration is provided via the [Restate Python SDK](https://github.com/restatedev/sdk-python/tree/main/python/restate/ext/pydantic).
+
+Visit the [Restate documentation](https://docs.restate.dev/ai/patterns/durable-agents) for more information.
+
+## Durable Execution
+
+Restate makes your agent **durable** by recording every step of its execution in a journal. If your process crashes mid-execution, Restate replays the journal, skips completed steps, and resumes from exactly where it left off.
+
+Your agent runs in a regular HTTP handler inside a Restate **service**. The Restate Server sits in front of your application and manages orchestration, journaling, and retries. Services run like regular Docker containers or serverless functions.
+
+A durable agent has three building blocks:
+
+1. The **handler**: your agent logic, exposed as an HTTP endpoint in a Restate service.
+2. **LLM calls**: persisted so responses are not re-fetched on recovery — saving cost and time.
+3. **Tool executions**: wrapped in durable steps so side effects are not duplicated.
+
+```text
+                  Clients
+              (HTTP, Kafka, etc.)
+                     |
+                     v
+            +---------------------+
+            |   Restate Server    |      (Journals execution,
+            +---------------------+       retries on failure,
+                     ^                    manages state)
+                     |
+        Journal      |   Replay on
+        steps,       |   recovery,
+        retries      |   schedule calls
+                     v
++------------------------------------------------------+
+|               Application Process                    |
+|   +----------------------------------------------+   |
+|   |         Restate Service Handler              |   |
+|   |           (Agent Run Loop)                   |   |
+|   |    [ Durable Steps (Tool, MCP, Model) ]      |   |
+|   +----------------------------------------------+   |
+|         |           |                |               |
++------------------------------------------------------+
+          |           |                |
+          v           v                v
+      [External APIs, services, databases, etc.]
+```
+
+See the [Restate documentation](https://docs.restate.dev/ai/patterns/durable-agents) for more information.
+
+## Durable Agent
+
+Any Pydantic AI agent can be made durable by wrapping it with `RestateAgent` from the Restate SDK and running it inside a Restate service handler.
+
+Install the Restate SDK:
+
+```bash
+pip/uv-add pydantic-ai "restate_sdk[serde]"
+```
+
+Here is a complete example of a durable Pydantic AI agent with Restate:
+
+```python {title="restate_agent.py" test="skip" lint="skip"}
+import restate
+from pydantic_ai import Agent, RunContext
+from restate.ext.pydantic import RestateAgent, restate_context
+
+weather_agent = Agent(  # (1)!
+    'openai:gpt-5.2',
+    system_prompt='You are a helpful agent that provides weather updates.',
+)
+
+
+@weather_agent.tool()
+async def get_weather(_run_ctx: RunContext, city: str) -> dict:
+    """Get the current weather for a given city."""
+
+    # Do durable tool steps using the Restate context
+    async def call_weather_api(city: str) -> dict:
+        return {'temperature': 23, 'description': 'Sunny and warm.'}
+
+    return await restate_context().run_typed(  # (2)!
+        f'Get weather {city}', call_weather_api, city=city
+    )
+
+
+restate_agent = RestateAgent(weather_agent)  # (3)!
+
+agent_service = restate.Service('WeatherAgent')
+
+
+@agent_service.handler()
+async def run(_ctx: restate.Context, prompt: str) -> str:  # (4)!
+    result = await restate_agent.run(prompt)
+    return result.output
+
+
+app = restate.app(services=[agent_service])  # (5)!
+
+if __name__ == "__main__":  # (6)!
+    import hypercorn
+    import asyncio
+    conf = hypercorn.Config()
+    conf.bind = ["0.0.0.0:9080"]
+    asyncio.run(hypercorn.asyncio.serve(app, conf))
+```
+
+1. Define your agent and tools as you normally would with Pydantic AI.
+2. Use `restate_context()` actions inside tools to make their execution durable. The result is persisted and retried until it succeeds. Side effects won't be duplicated on recovery.
+3. `RestateAgent` wraps the agent so every LLM response is saved in the Restate Server and replayed during recovery.
+4. The Restate service handler gives the agent a durable execution context and exposes it as an HTTP endpoint.
+5. `restate.app()` creates the application that can be served.
+6. Run the application with an ASGI server like Hypercorn.
+
+See the [Restate agent quickstart](https://docs.restate.dev/ai-quickstart) to learn how to run the agent.

docs/durable_execution/temporal.md

@@ -0,0 +1,328 @@
+# Durable Execution with Temporal
+
+[Temporal](https://temporal.io) is a popular [durable execution](https://docs.temporal.io/evaluate/understanding-temporal#durable-execution) platform that's natively supported by Pydantic AI.
+
+## Durable Execution
+
+In Temporal's durable execution implementation, a program that crashes or encounters an exception while interacting with a model or API will retry until it can successfully complete.
+
+Temporal relies primarily on a replay mechanism to recover from failures.
+As the program makes progress, Temporal saves key inputs and decisions, allowing a re-started program to pick up right where it left off.
+
+The key to making this work is to separate the application's repeatable (deterministic) and non-repeatable (non-deterministic) parts:
+
+1. Deterministic pieces, termed [**workflows**](https://docs.temporal.io/workflow-definition), execute the same way when re-run with the same inputs.
+2. Non-deterministic pieces, termed [**activities**](https://docs.temporal.io/activities), can run arbitrary code, performing I/O and any other operations.
+
+Workflow code can run for extended periods and, if interrupted, resume exactly where it left off.
+Critically, workflow code generally _cannot_ include any kind of I/O, over the network, disk, etc.
+Activity code faces no restrictions on I/O or external interactions, but if an activity fails part-way through it is restarted from the beginning.
+
+
+!!! note
+
+    If you are familiar with celery, it may be helpful to think of Temporal activities as similar to celery tasks, but where you wait for the task to complete and obtain its result before proceeding to the next step in the workflow.
+    However, Temporal workflows and activities offer a great deal more flexibility and functionality than celery tasks.
+
+    See the [Temporal documentation](https://docs.temporal.io/evaluate/understanding-temporal#temporal-application-the-building-blocks) for more information
+
+In the case of Pydantic AI agents, integration with Temporal means that [model requests](../models/overview.md), [tool calls](../tools.md) that may require I/O, and [MCP server communication](../mcp/client.md) all need to be offloaded to Temporal activities due to their I/O requirements, while the logic that coordinates them (i.e. the agent run) lives in the workflow. Code that handles a scheduled job or web request can then execute the workflow, which will in turn execute the activities as needed.
+
+The diagram below shows the overall architecture of an agentic application in Temporal.
+The Temporal Server is responsible for tracking program execution and making sure the associated state is preserved reliably (i.e., stored to an internal database, and possibly replicated across cloud regions).
+Temporal Server manages data in encrypted form, so all data processing occurs on the Worker, which runs the workflow and activities.
+
+
+```text
+            +---------------------+
+            |   Temporal Server   |      (Stores workflow state,
+            +---------------------+       schedules activities,
+                     ^                    persists progress)
+                     |
+        Save state,  |   Schedule Tasks,
+        progress,    |   load state on resume
+        timeouts     |
+                     |
++------------------------------------------------------+
+|                      Worker                          |
+|   +----------------------------------------------+   |
+|   |              Workflow Code                   |   |
+|   |       (Agent Run Loop)                       |   |
+|   +----------------------------------------------+   |
+|          |          |                |               |
+|          v          v                v               |
+|   +-----------+ +------------+ +-------------+       |
+|   | Activity  | | Activity   | |  Activity   |       |
+|   | (Tool)    | | (MCP Tool) | | (Model API) |       |
+|   +-----------+ +------------+ +-------------+       |
+|         |           |                |               |
++------------------------------------------------------+
+          |           |                |
+          v           v                v
+      [External APIs, services, databases, etc.]
+```
+
+See the [Temporal documentation](https://docs.temporal.io/evaluate/understanding-temporal#temporal-application-the-building-blocks) for more information.
+
+## Durable Agent
+
+Any agent can be wrapped in a [`TemporalAgent`][pydantic_ai.durable_exec.temporal.TemporalAgent] to get a durable agent that can be used inside a deterministic Temporal workflow, by automatically offloading all work that requires I/O (namely model requests, tool calls, and MCP server communication) to non-deterministic activities.
+
+At the time of wrapping, the agent's [model](../models/overview.md) and [toolsets](../toolsets.md) (including function tools registered on the agent and MCP servers) are frozen, activities are dynamically created for each, and the original model and toolsets are wrapped to call on the worker to execute the corresponding activities instead of directly performing the actions inside the workflow. The original agent can still be used as normal outside the Temporal workflow, but any changes to its model or toolsets after wrapping will not be reflected in the durable agent.
+
+Here is a simple but complete example of wrapping an agent for durable execution, creating a Temporal workflow with durable execution logic, connecting to a Temporal server, and running the workflow from non-durable code. All it requires is a Temporal server to be [running locally](https://github.com/temporalio/temporal#download-and-start-temporal-server-locally):
+
+```sh
+brew install temporal
+temporal server start-dev
+```
+
+```python {title="temporal_agent.py" test="skip"}
+import uuid
+
+from temporalio import workflow
+from temporalio.client import Client
+from temporalio.worker import Worker
+
+from pydantic_ai import Agent
+from pydantic_ai.durable_exec.temporal import (
+    PydanticAIPlugin,
+    PydanticAIWorkflow,
+    TemporalAgent,
+)
+
+agent = Agent(
+    'openai:gpt-5.2',
+    instructions="You're an expert in geography.",
+    name='geography',  # (10)!
+)
+
+temporal_agent = TemporalAgent(agent)  # (1)!
+
+
+@workflow.defn
+class GeographyWorkflow(PydanticAIWorkflow):  # (2)!
+    __pydantic_ai_agents__ = [temporal_agent]  # (3)!
+
+    @workflow.run
+    async def run(self, prompt: str) -> str:
+        result = await temporal_agent.run(prompt)  # (4)!
+        return result.output
+
+
+async def main():
+    client = await Client.connect(  # (5)!
+        'localhost:7233',  # (6)!
+        plugins=[PydanticAIPlugin()],  # (7)!
+    )
+
+    async with Worker(  # (8)!
+        client,
+        task_queue='geography',
+        workflows=[GeographyWorkflow],
+    ):
+        output = await client.execute_workflow(  # (10)!
+            GeographyWorkflow.run,
+            args=['What is the capital of Mexico?'],
+            id=f'geography-{uuid.uuid4()}',
+            task_queue='geography',
+        )
+        print(output)
+        #> Mexico City (Ciudad de México, CDMX)
+```
+
+1. The original `Agent` cannot be used inside a deterministic Temporal workflow, but the `TemporalAgent` can.
+2. As explained above, the workflow represents a deterministic piece of code that can use non-deterministic activities for operations that require I/O. Subclassing [`PydanticAIWorkflow`][pydantic_ai.durable_exec.temporal.PydanticAIWorkflow] is optional but provides proper typing for the `__pydantic_ai_agents__` class variable.
+3. List the `TemporalAgent`s used by this workflow. The [`PydanticAIPlugin`][pydantic_ai.durable_exec.temporal.PydanticAIPlugin] will automatically register their activities with the worker. Alternatively, if modifying the worker initialization is easier than the workflow class, you can use [`AgentPlugin`][pydantic_ai.durable_exec.temporal.AgentPlugin] to register agents directly on the worker.
+4. [`TemporalAgent.run()`][pydantic_ai.durable_exec.temporal.TemporalAgent.run] works just like [`Agent.run()`][pydantic_ai.agent.Agent.run], but it will automatically offload model requests, tool calls, and MCP server communication to Temporal activities.
+5. We connect to the Temporal server which keeps track of workflow and activity execution.
+6. This assumes the Temporal server is [running locally](https://github.com/temporalio/temporal#download-and-start-temporal-server-locally).
+7. The [`PydanticAIPlugin`][pydantic_ai.durable_exec.temporal.PydanticAIPlugin] tells Temporal to use Pydantic for serialization and deserialization, treats [`UserError`][pydantic_ai.exceptions.UserError] exceptions as non-retryable, and automatically registers activities for agents listed in `__pydantic_ai_agents__`.
+8. We start the worker that will listen on the specified task queue and run workflows and activities. In a real world application, this might be run in a separate service.
+9. The agent's `name` is used to uniquely identify its activities.
+10. We call on the server to execute the workflow on a worker that's listening on the specified task queue.
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+In a real world application, the agent, workflow, and worker are typically defined separately from the code that calls for a workflow to be executed.
+Because Temporal workflows need to be defined at the top level of the file and the `TemporalAgent` instance is needed inside the workflow and when starting the worker (to register the activities), it needs to be defined at the top level of the file as well.
+
+For more information on how to use Temporal in Python applications, see their [Python SDK guide](https://docs.temporal.io/develop/python).
+
+## Temporal Integration Considerations
+
+There are a few considerations specific to agents and toolsets when using Temporal for durable execution. These are important to understand to ensure that your agents and toolsets work correctly with Temporal's workflow and activity model.
+
+### Agent Names and Toolset IDs
+
+To ensure that Temporal knows what code to run when an activity fails or is interrupted and then restarted, even if your code is changed in between, each activity needs to have a name that's stable and unique.
+
+When `TemporalAgent` dynamically creates activities for the wrapped agent's model requests and toolsets (specifically those that implement their own tool listing and calling, i.e. [`FunctionToolset`][pydantic_ai.toolsets.FunctionToolset] and [`MCPToolset`][pydantic_ai.mcp.MCPToolset]), their names are derived from the agent's [`name`][pydantic_ai.agent.AbstractAgent.name] and the toolsets' [`id`s][pydantic_ai.toolsets.AbstractToolset.id]. These fields are normally optional, but are required to be set when using Temporal. They should not be changed once the durable agent has been deployed to production as this would break active workflows.
+
+For dynamic toolsets created with the [`@agent.toolset`][pydantic_ai.agent.Agent.toolset] decorator, the `id` parameter must be set explicitly. Note that with Temporal, `per_run_step=False` is not respected, as the toolset always needs to be created on-the-fly in the activity.
+
+Other than that, any agent and toolset will just work!
+
+### Agent Run Context and Dependencies
+
+As workflows and activities run in separate processes, any values passed between them need to be serializable. As these payloads are stored in the workflow execution event history, Temporal limits their size to 2MB.
+
+To account for these limitations, tool functions and the [event stream handler](#streaming) running inside activities receive a limited version of the agent's [`RunContext`][pydantic_ai.tools.RunContext], and it's your responsibility to make sure that the [dependencies](../dependencies.md) object provided to [`TemporalAgent.run()`][pydantic_ai.durable_exec.temporal.TemporalAgent.run] can be serialized using Pydantic.
+
+Specifically, only the `deps`, `run_id`, `metadata`, `retries`, `tool_call_id`, `tool_name`, `tool_call_approved`, `tool_call_metadata`, `retry`, `max_retries`, `run_step`, `usage`, and `partial_output` fields are available by default, and trying to access `model`, `prompt`, `messages`, or `tracer` will raise an error.
+If you need one or more of these attributes to be available inside activities, you can create a [`TemporalRunContext`][pydantic_ai.durable_exec.temporal.TemporalRunContext] subclass with custom `serialize_run_context` and `deserialize_run_context` class methods and pass it to [`TemporalAgent`][pydantic_ai.durable_exec.temporal.TemporalAgent] as `run_context_type`.
+
+### Streaming
+
+Because Temporal activities cannot stream output directly to the activity call site, [`Agent.run_stream()`][pydantic_ai.agent.Agent.run_stream], [`Agent.run_stream_events()`][pydantic_ai.agent.Agent.run_stream_events], and [`Agent.iter()`][pydantic_ai.agent.Agent.iter] are not supported.
+
+Instead, you can implement streaming by setting an [`event_stream_handler`][pydantic_ai.agent.EventStreamHandler] on the `Agent` or `TemporalAgent` instance and using [`TemporalAgent.run()`][pydantic_ai.durable_exec.temporal.TemporalAgent.run] inside the workflow.
+The event stream handler function will receive the agent [run context][pydantic_ai.tools.RunContext] and an async iterable of events from the model's streaming response and the agent's execution of tools. For examples, see the [streaming docs](../agent.md#streaming-all-events).
+
+As the streaming model request activity, workflow, and workflow execution call all take place in separate processes, passing data between them requires some care:
+
+- To get data from the workflow call site or workflow to the event stream handler, you can use a [dependencies object](#agent-run-context-and-dependencies).
+- To get data from the event stream handler to the workflow, workflow call site, or a frontend, you need to use an external system that the event stream handler can write to and the event consumer can read from, like a message queue. You can use the dependency object to make sure the same connection string or other unique ID is available in all the places that need it.
+
+### Model Selection at Runtime
+
+[`Agent.run(model=...)`][pydantic_ai.agent.Agent.run] normally supports both model strings (like `'openai:gpt-5.2'`) and model instances. However, `TemporalAgent` does not support arbitrary model instances because they cannot be serialized for Temporal's replay mechanism.
+
+To use model instances with `TemporalAgent`, you need to pre-register them by passing a dict of model instances to `TemporalAgent(models={...})`. You can then reference them by name or by passing the registered instance directly. If the wrapped agent doesn't have a model set, the first registered model will be used as the default.
+
+Model strings work as expected. For scenarios where you need to customize the provider used by the model string (e.g., inject API keys from deps), you can pass a `provider_factory` to `TemporalAgent`, which is passed the [`RunContext`][pydantic_ai.tools.RunContext] and provider name.
+
+Here's an example showing how to pre-register and use multiple models:
+
+```python {title="multi_model_temporal.py" test="skip"}
+from dataclasses import dataclass
+from typing import Any
+
+from temporalio import workflow
+
+from pydantic_ai import Agent, RunContext
+from pydantic_ai.durable_exec.temporal import TemporalAgent
+from pydantic_ai.models.anthropic import AnthropicModel
+from pydantic_ai.models.google import GoogleModel
+from pydantic_ai.models.openai import OpenAIResponsesModel
+from pydantic_ai.providers import Provider
+
+
+@dataclass
+class Deps:
+    openai_api_key: str | None = None
+    anthropic_api_key: str | None = None
+
+
+# Create models from different providers
+default_model = OpenAIResponsesModel('gpt-5.2')
+fast_model = AnthropicModel('claude-sonnet-4-5')
+reasoning_model = GoogleModel('gemini-3-pro-preview')
+
+
+# Optional: provider factory for dynamic model configuration
+def my_provider_factory(run_context: RunContext[Deps], provider_name: str) -> Provider[Any]:
+    """Create providers with custom configuration based on run context."""
+    if provider_name == 'openai':
+        from pydantic_ai.providers.openai import OpenAIProvider
+
+        return OpenAIProvider(api_key=run_context.deps.openai_api_key)
+    elif provider_name == 'anthropic':
+        from pydantic_ai.providers.anthropic import AnthropicProvider
+
+        return AnthropicProvider(api_key=run_context.deps.anthropic_api_key)
+    else:
+        raise ValueError(f'Unknown provider: {provider_name}')
+
+
+agent = Agent(default_model, name='multi_model_agent', deps_type=Deps)
+
+temporal_agent = TemporalAgent(
+    agent,
+    models={
+        'fast': fast_model,
+        'reasoning': reasoning_model,
+    },
+    provider_factory=my_provider_factory,  # Optional
+)
+
+
+@workflow.defn
+class MultiModelWorkflow:
+    @workflow.run
+    async def run(self, prompt: str, use_reasoning: bool, use_fast: bool) -> str:
+        if use_reasoning:
+            # Select by registered name
+            result = await temporal_agent.run(prompt, model='reasoning')
+        elif use_fast:
+            # Or pass the registered instance directly
+            result = await temporal_agent.run(prompt, model=fast_model)
+        else:
+            # Or pass a model string (uses provider_factory if set)
+            result = await temporal_agent.run(prompt, model='openai:gpt-5-mini')
+        return result.output
+```
+
+### Toolsets at Runtime
+
+Additional toolsets can be passed per run via [`TemporalAgent.run(toolsets=...)`][pydantic_ai.durable_exec.temporal.TemporalAgent.run], but only non-executing toolsets like [`ExternalToolset`][pydantic_ai.toolsets.ExternalToolset], whose tools are executed outside the agent run, are supported. Executing toolsets ([`FunctionToolset`][pydantic_ai.toolsets.FunctionToolset] and [`MCPToolset`][pydantic_ai.mcp.MCPToolset]) and dynamic toolsets must be set when constructing the agent so their activities can be registered with the worker before the workflow runs; passing them at runtime raises a `UserError`.
+
+## Activity Configuration
+
+Temporal activity configuration, like timeouts and retry policies, can be customized by passing [`temporalio.workflow.ActivityConfig`](https://python.temporal.io/temporalio.workflow._activities.ActivityConfig.html) objects to the `TemporalAgent` constructor:
+
+- `activity_config`: The base Temporal activity config to use for all activities. If no config is provided, a `start_to_close_timeout` of 60 seconds is used.
+- `model_activity_config`: The Temporal activity config to use for model request activities. This is merged with the base activity config.
+- `toolset_activity_config`: The Temporal activity config to use for get-tools and call-tool activities for specific toolsets identified by ID. This is merged with the base activity config.
+- `tool_activity_config`: The Temporal activity config to use for specific tool call activities identified by toolset ID and tool name.
+    This is merged with the base and toolset-specific activity configs.
+
+    If a tool does not use I/O, you can specify `False` to disable using an activity. Note that the tool is required to be defined as an `async` function as non-async tools are run in threads which are non-deterministic and thus not supported outside of activities.
+
+## Activity Retries
+
+On top of the automatic retries for request failures that Temporal will perform, Pydantic AI and various provider API clients also have their own request retry logic. Enabling these at the same time may cause the request to be retried more often than expected, with improper `Retry-After` handling.
+
+When using Temporal, it's recommended to not use [HTTP Request Retries](../retries.md) and to turn off your provider API client's own retry logic, for example by setting `max_retries=0` on a [custom `OpenAIProvider` API client](../models/openai.md#custom-openai-client).
+
+You can customize Temporal's retry policy using [activity configuration](#activity-configuration).
+
+## Observability with Logfire
+
+Temporal generates telemetry events and metrics for each workflow and activity execution, and Pydantic AI generates events for each agent run, model request and tool call. These can be sent to [Pydantic Logfire](../logfire.md) to get a complete picture of what's happening in your application.
+
+To use Logfire with Temporal, you need to pass a [`LogfirePlugin`][pydantic_ai.durable_exec.temporal.LogfirePlugin] object to Temporal's `Client.connect()`:
+
+```py {title="logfire_plugin.py" test="skip" noqa="F841"}
+from temporalio.client import Client
+
+from pydantic_ai.durable_exec.temporal import LogfirePlugin, PydanticAIPlugin
+
+
+async def main():
+    client = await Client.connect(
+        'localhost:7233',
+        plugins=[PydanticAIPlugin(), LogfirePlugin()],
+    )
+```
+
+By default, the `LogfirePlugin` will instrument Temporal (including metrics) and Pydantic AI and send all data to Logfire. To customize Logfire configuration and instrumentation, you can pass a `logfire_setup` function to the `LogfirePlugin` constructor and return a custom `Logfire` instance (i.e. the result of `logfire.configure()`). To disable sending Temporal metrics to Logfire, you can pass `metrics=False` to the `LogfirePlugin` constructor.
+
+## Known Issues
+
+### Pandas
+
+When `logfire.info` is used inside an activity and the `pandas` package is among your project's dependencies, you may encounter the following error which seems to be the result of an import race condition:
+
+```
+AttributeError: partially initialized module 'pandas' has no attribute '_pandas_parser_CAPI' (most likely due to a circular import)
+```
+
+To fix this, you can use the [`temporalio.workflow.unsafe.imports_passed_through()`](https://python.temporal.io/temporalio.workflow._sandbox.unsafe.html#imports_passed_through) context manager to proactively import the package and not have it be reloaded in the workflow sandbox:
+
+```python {title="temporal_activity.py" test="skip" noqa="F401"}
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    import pandas
+```

docs/embeddings.md

@@ -0,0 +1,883 @@
+# Embeddings
+
+Embeddings are vector representations of text that capture semantic meaning. They're essential for building:
+
+- **Semantic search** — Find documents based on meaning, not just keyword matching
+- **RAG (Retrieval-Augmented Generation)** — Retrieve relevant context for your AI agents
+- **Similarity detection** — Find similar documents, detect duplicates, or cluster content
+- **Classification** — Use embeddings as features for downstream ML models
+
+Pydantic AI provides a unified interface for generating embeddings across multiple providers.
+
+## Quick Start
+
+The [`Embedder`][pydantic_ai.embeddings.Embedder] class is the high-level interface for generating embeddings:
+
+```python {title="embeddings_quickstart.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('openai:text-embedding-3-small')
+
+
+async def main():
+    # Embed a search query
+    result = await embedder.embed_query('What is machine learning?')
+    print(f'Embedding dimensions: {len(result.embeddings[0])}')
+    #> Embedding dimensions: 1536
+
+    # Embed multiple documents at once
+    docs = [
+        'Machine learning is a subset of AI.',
+        'Deep learning uses neural networks.',
+        'Python is a programming language.',
+    ]
+    result = await embedder.embed_documents(docs)
+    print(f'Embedded {len(result.embeddings)} documents')
+    #> Embedded 3 documents
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+!!! tip "Queries vs Documents"
+    Some embedding models optimize differently for queries and documents. Use
+    [`embed_query()`][pydantic_ai.embeddings.Embedder.embed_query] for search queries and
+    [`embed_documents()`][pydantic_ai.embeddings.Embedder.embed_documents] for content you're indexing.
+
+## Embedding Result
+
+All embed methods return an [`EmbeddingResult`][pydantic_ai.embeddings.EmbeddingResult] containing the embeddings along with useful metadata.
+
+For convenience, you can access embeddings either by index (`result[0]`) or by the original input text (`result['Hello world']`).
+
+```python {title="embedding_result.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('openai:text-embedding-3-small')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+
+    # Access embeddings - each is a sequence of floats
+    embedding = result.embeddings[0]  # By index via .embeddings
+    embedding = result[0]  # Or directly via __getitem__
+    embedding = result['Hello world']  # Or by original input text
+    print(f'Dimensions: {len(embedding)}')
+    #> Dimensions: 1536
+
+    # Check usage
+    print(f'Tokens used: {result.usage.input_tokens}')
+    #> Tokens used: 2
+
+    # Calculate cost (requires `genai-prices` to have pricing data for the model)
+    cost = result.cost()
+    print(f'Cost: ${cost.total_price:.6f}')
+    #> Cost: $0.000000
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Choosing a model
+
+The best embedding model depends on your constraints. Here's a starting-point cheat sheet; consult each provider's docs and the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard) before committing to a model for a large index.
+
+| If you want…                         | For example                                                                                                                                                                                                                                      |
+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| A managed API                        | `openai:text-embedding-3-small` (cheap default), `openai:text-embedding-3-large`, `voyageai:voyage-3.5`, or `cohere:embed-v4.0`                                                                                                                  |
+| No API key, private, free            | `sentence-transformers:google/embeddinggemma-300m`, `sentence-transformers:lightonai/DenseOn`, `sentence-transformers:Qwen/Qwen3-Embedding-0.6B`, or any other [Hugging Face model](https://huggingface.co/models?library=sentence-transformers) |
+| Multilingual                         | `cohere:embed-multilingual-v3.0`, `sentence-transformers:jinaai/jina-embeddings-v5-text-small-retrieval`, or `sentence-transformers:Snowflake/snowflake-arctic-embed-l-v2.0`                                                                     |
+| Specialized domain                   | `voyageai:voyage-code-3`, `voyageai:voyage-law-2`, `voyageai:voyage-finance-2`, `sentence-transformers:nomic-ai/CodeRankEmbed`, or `sentence-transformers:TechWolf/JobBERT-v3`                                                                   |
+| To run on AWS infra you already have | `bedrock:amazon.titan-embed-text-v2:0` or `bedrock:cohere.embed-v4:0`                                                                                                                                                                            |
+| To reduce index size                 | Any model with dimension control (see [Settings](#settings))                                                                                                                                                                                     |
+
+!!! tip "Switching models later"
+    Swapping a model changes the output dimension and the similarity distribution, so you'll need to re-embed (and re-index) your documents. Pick a model you're happy to stick with, or one that supports [dimension control](#settings) so you can tune the index size without changing models.
+
+## Providers
+
+### OpenAI
+
+[`OpenAIEmbeddingModel`][pydantic_ai.embeddings.openai.OpenAIEmbeddingModel] works with OpenAI's embeddings API and any [OpenAI-compatible provider](models/openai.md#openai-compatible-models).
+
+#### Install
+
+To use OpenAI embedding models, you need to either install `pydantic-ai`, or install `pydantic-ai-slim` with the `openai` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[openai]"
+```
+
+#### Configuration
+
+To use `OpenAIEmbeddingModel` with the OpenAI API, go to [platform.openai.com](https://platform.openai.com/) and follow your nose until you find the place to generate an API key. Once you have the API key, you can set it as an environment variable:
+
+```bash
+export OPENAI_API_KEY='your-api-key'
+```
+
+You can then use the model:
+
+```python {title="openai_embeddings.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('openai:text-embedding-3-small')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 1536
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+See [OpenAI's embedding models](https://platform.openai.com/docs/guides/embeddings) for available models.
+
+#### Dimension Control
+
+OpenAI's `text-embedding-3-*` models support dimension reduction via the `dimensions` setting:
+
+```python {title="openai_dimensions.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings import EmbeddingSettings
+
+embedder = Embedder(
+    'openai:text-embedding-3-small',
+    settings=EmbeddingSettings(dimensions=256),
+)
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 256
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+#### OpenAI-Compatible Providers {#openai-compatible}
+
+Since [`OpenAIEmbeddingModel`][pydantic_ai.embeddings.openai.OpenAIEmbeddingModel] uses the same provider system as [`OpenAIChatModel`][pydantic_ai.models.openai.OpenAIChatModel], you can use it with any [OpenAI-compatible provider](models/openai.md#openai-compatible-models):
+
+```python {title="openai_compatible_embeddings.py"}
+# Using Azure OpenAI
+from openai import AsyncAzureOpenAI
+
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.openai import OpenAIEmbeddingModel
+from pydantic_ai.providers.openai import OpenAIProvider
+
+azure_client = AsyncAzureOpenAI(
+    azure_endpoint='https://your-resource.openai.azure.com',
+    api_version='2024-02-01',
+    api_key='your-azure-key',
+)
+model = OpenAIEmbeddingModel(
+    'text-embedding-3-small',
+    provider=OpenAIProvider(openai_client=azure_client),
+)
+embedder = Embedder(model)
+
+
+# Using any OpenAI-compatible API
+model = OpenAIEmbeddingModel(
+    'your-model-name',
+    provider=OpenAIProvider(
+        base_url='https://your-provider.com/v1',
+        api_key='your-api-key',
+    ),
+)
+embedder = Embedder(model)
+```
+
+For providers with dedicated provider classes (like [`OllamaProvider`][pydantic_ai.providers.ollama.OllamaProvider] or [`AzureProvider`][pydantic_ai.providers.azure.AzureProvider]), you can use the shorthand syntax:
+
+```python
+from pydantic_ai import Embedder
+
+embedder = Embedder('azure:text-embedding-3-small')
+embedder = Embedder('ollama:nomic-embed-text')
+```
+
+See [OpenAI-compatible Models](models/openai.md#openai-compatible-models) for the full list of supported providers.
+
+### Google
+
+[`GoogleEmbeddingModel`][pydantic_ai.embeddings.google.GoogleEmbeddingModel] works with Google's embedding models via the Gemini API (Google AI Studio) or Google Cloud (formerly known as Vertex AI).
+
+#### Install
+
+To use Google embedding models, you need to either install `pydantic-ai`, or install `pydantic-ai-slim` with the `google` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[google]"
+```
+
+#### Configuration
+
+To use `GoogleEmbeddingModel` with the Gemini API, go to [aistudio.google.com](https://aistudio.google.com/) and generate an API key. Once you have the API key, you can set it as an environment variable:
+
+```bash
+export GOOGLE_API_KEY='your-api-key'
+```
+
+You can then use the model:
+
+```python {title="google_embeddings.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('google:gemini-embedding-001')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 3072
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+See the [Google Embeddings documentation](https://ai.google.dev/gemini-api/docs/embeddings) for available models.
+
+##### Google Cloud
+
+To use Google's embedding models via Google Cloud (formerly known as Vertex AI) instead of the Gemini API, use the `google-cloud:` provider prefix:
+
+```python {title="google_cloud_embeddings.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.google import GoogleEmbeddingModel
+from pydantic_ai.providers.google_cloud import GoogleCloudProvider
+
+# Using provider prefix
+embedder = Embedder('google-cloud:gemini-embedding-001')
+
+# Or with explicit provider configuration
+model = GoogleEmbeddingModel(
+    'gemini-embedding-001',
+    provider=GoogleCloudProvider(project='my-project', location='us-central1'),
+)
+embedder = Embedder(model)
+```
+
+See the [Google provider documentation](models/google.md#google-cloud-enterprise) for more details on Google Cloud authentication options, including application default credentials, service accounts, and API keys.
+
+#### Dimension Control
+
+Google's embedding models support dimension reduction via the `dimensions` setting:
+
+```python {title="google_dimensions.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings import EmbeddingSettings
+
+embedder = Embedder(
+    'google:gemini-embedding-001',
+    settings=EmbeddingSettings(dimensions=768),
+)
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 768
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+#### Task Conditioning
+
+`gemini-embedding-2` is conditioned on the task you're embedding for by prepending a short task instruction to the input text, rather than through the [`google_task_type`][pydantic_ai.embeddings.google.GoogleEmbeddingSettings.google_task_type] field used by the other Google models. Pydantic AI builds this prefix for you via the `google_task` setting:
+
+```python {title="google_task.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.google import GoogleEmbeddingSettings
+
+embedder = Embedder(
+    'google:gemini-embedding-2',
+    settings=GoogleEmbeddingSettings(google_task='question answering'),
+)
+```
+
+`google_task` accepts the task names from Google's API: `'search result'`, `'question answering'`, `'fact checking'`, `'code retrieval'`, `'classification'`, `'clustering'`, and `'sentence similarity'`. For the retrieval-style (asymmetric) tasks, queries and documents are prefixed differently, so the same task applies to both sides of a pair; the remaining (symmetric) tasks prefix both inputs the same way.
+
+When you don't set `google_task`, `gemini-embedding-2` is conditioned as `'search result'`. Conditioning is on by default because Google recommends it for this model and it yields better retrieval performance than embedding raw text. To opt out and embed the text verbatim, pass `google_task='raw'`.
+
+`google_task` only applies to `gemini-embedding-2`; on any other model it is ignored with a warning (those models condition via `google_task_type` instead). Conversely, `google_task_type` is ignored on `gemini-embedding-2`, since that model conditions through the text prefix.
+
+#### Google-Specific Settings
+
+Google models support additional settings via [`GoogleEmbeddingSettings`][pydantic_ai.embeddings.google.GoogleEmbeddingSettings]:
+
+```python {title="google_settings.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.google import GoogleEmbeddingSettings
+
+embedder = Embedder(
+    'google:gemini-embedding-001',
+    settings=GoogleEmbeddingSettings(
+        dimensions=768,
+        google_task_type='SEMANTIC_SIMILARITY',  # Optimize for similarity comparison
+    ),
+)
+```
+
+See [Google's task type documentation](https://ai.google.dev/gemini-api/docs/embeddings#task-types) for available task types. By default, `embed_query()` uses `RETRIEVAL_QUERY` and `embed_documents()` uses `RETRIEVAL_DOCUMENT`.
+
+### Cohere
+
+[`CohereEmbeddingModel`][pydantic_ai.embeddings.cohere.CohereEmbeddingModel] provides access to Cohere's embedding models, which offer multilingual support and various model sizes.
+
+#### Install
+
+To use Cohere embedding models, you need to either install `pydantic-ai`, or install `pydantic-ai-slim` with the `cohere` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[cohere]"
+```
+
+#### Configuration
+
+To use `CohereEmbeddingModel`, go to [dashboard.cohere.com/api-keys](https://dashboard.cohere.com/api-keys) and follow your nose until you find the place to generate an API key. Once you have the API key, you can set it as an environment variable:
+
+```bash
+export CO_API_KEY='your-api-key'
+```
+
+You can then use the model:
+
+```python {title="cohere_embeddings.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('cohere:embed-v4.0')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 1024
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+See the [Cohere Embed documentation](https://docs.cohere.com/docs/cohere-embed) for available models.
+
+#### Cohere-Specific Settings
+
+Cohere models support additional settings via [`CohereEmbeddingSettings`][pydantic_ai.embeddings.cohere.CohereEmbeddingSettings]:
+
+```python {title="cohere_settings.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.cohere import CohereEmbeddingSettings
+
+embedder = Embedder(
+    'cohere:embed-v4.0',
+    settings=CohereEmbeddingSettings(
+        dimensions=512,
+        cohere_truncate='END',  # Truncate long inputs instead of erroring
+        cohere_max_tokens=256,  # Limit tokens per input
+    ),
+)
+```
+
+### VoyageAI
+
+[`VoyageAIEmbeddingModel`][pydantic_ai.embeddings.voyageai.VoyageAIEmbeddingModel] provides access to VoyageAI's embedding models, which are optimized for retrieval with specialized models for code, finance, and legal domains.
+
+#### Install
+
+To use VoyageAI embedding models, you need to install `pydantic-ai-slim` with the `voyageai` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[voyageai]"
+```
+
+#### Configuration
+
+To use `VoyageAIEmbeddingModel`, go to [dash.voyageai.com](https://dash.voyageai.com/) to generate an API key. Once you have the API key, you can set it as an environment variable:
+
+```bash
+export VOYAGE_API_KEY='your-api-key'
+```
+
+You can then use the model:
+
+```python {title="voyageai_embeddings.py" max_py="3.13"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('voyageai:voyage-3.5')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 1024
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+See the [VoyageAI Embeddings documentation](https://docs.voyageai.com/docs/embeddings) for available models.
+
+#### VoyageAI-Specific Settings
+
+VoyageAI models support additional settings via [`VoyageAIEmbeddingSettings`][pydantic_ai.embeddings.voyageai.VoyageAIEmbeddingSettings]:
+
+```python {title="voyageai_settings.py" max_py="3.13"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.voyageai import VoyageAIEmbeddingSettings
+
+embedder = Embedder(
+    'voyageai:voyage-3.5',
+    settings=VoyageAIEmbeddingSettings(
+        dimensions=512,  # Reduce output dimensions
+        voyageai_input_type='document',  # Override input type for all requests
+    ),
+)
+```
+
+### Bedrock
+
+[`BedrockEmbeddingModel`][pydantic_ai.embeddings.bedrock.BedrockEmbeddingModel] provides access to embedding models through AWS Bedrock, including Amazon Titan, Cohere, and Amazon Nova models.
+
+#### Install
+
+To use Bedrock embedding models, you need to either install `pydantic-ai`, or install `pydantic-ai-slim` with the `bedrock` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[bedrock]"
+```
+
+#### Configuration
+
+Authentication with AWS Bedrock uses standard AWS credentials. See the [Bedrock provider documentation](models/bedrock.md#environment-variables) for details on configuring credentials via environment variables, AWS credentials file, or IAM roles.
+
+Ensure your AWS account has access to the Bedrock embedding models you want to use. See [AWS Bedrock model access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) for details.
+
+#### Basic Usage
+
+```python {title="bedrock_embeddings.py" test="skip"}
+from pydantic_ai import Embedder
+
+# Using Amazon Titan
+embedder = Embedder('bedrock:amazon.titan-embed-text-v2:0')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 1024
+```
+
+_(This example requires AWS credentials configured)_
+
+#### Supported Models
+
+Bedrock supports three families of embedding models. See the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html) for the full list of available models.
+
+**Amazon Titan:**
+
+- `amazon.titan-embed-text-v1` — 1536 dimensions (fixed), 8K tokens
+- `amazon.titan-embed-text-v2:0` — 256/384/1024 dimensions (configurable, default: 1024), 8K tokens
+
+**Cohere Embed:**
+
+- `cohere.embed-english-v3` — English-only, 1024 dimensions (fixed), 512 tokens
+- `cohere.embed-multilingual-v3` — Multilingual, 1024 dimensions (fixed), 512 tokens
+- `cohere.embed-v4:0` — 256/512/1024/1536 dimensions (configurable, default: 1536), 128K tokens
+
+**Amazon Nova:**
+
+- `amazon.nova-2-multimodal-embeddings-v1:0` — 256/384/1024/3072 dimensions (configurable, default: 3072), 8K tokens
+
+#### Titan-Specific Settings
+
+Titan v2 supports vector normalization for direct similarity calculations via `bedrock_titan_normalize` (default: `True`). Titan v1 does not support this setting.
+
+```python {title="bedrock_titan.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingSettings
+
+embedder = Embedder(
+    'bedrock:amazon.titan-embed-text-v2:0',
+    settings=BedrockEmbeddingSettings(
+        dimensions=512,
+        bedrock_titan_normalize=True,
+    ),
+)
+```
+
+!!! note
+    Titan models do not support the `truncate` setting. The `dimensions` setting is only supported by Titan v2.
+
+#### Cohere-Specific Settings
+
+Cohere models on Bedrock support additional settings via [`BedrockEmbeddingSettings`][pydantic_ai.embeddings.bedrock.BedrockEmbeddingSettings]:
+
+- `bedrock_cohere_input_type` — By default, `embed_query()` uses `'search_query'` and `embed_documents()` uses `'search_document'`. Also accepts `'classification'` or `'clustering'`.
+- `bedrock_cohere_truncate` — Fine-grained truncation control: `'NONE'` (default, error on overflow), `'START'`, or `'END'`. Overrides the base `truncate` setting.
+- `bedrock_cohere_max_tokens` — Limits tokens per input (default: 128000). Only supported by Cohere v4.
+
+```python {title="bedrock_cohere.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingSettings
+
+embedder = Embedder(
+    'bedrock:cohere.embed-v4:0',
+    settings=BedrockEmbeddingSettings(
+        dimensions=512,
+        bedrock_cohere_max_tokens=1000,
+        bedrock_cohere_truncate='END',
+    ),
+)
+```
+
+!!! note
+    The `dimensions` and `bedrock_cohere_max_tokens` settings are only supported by Cohere v4. Cohere v3 models have fixed 1024 dimensions.
+
+#### Nova-Specific Settings
+
+Nova models on Bedrock support additional settings via [`BedrockEmbeddingSettings`][pydantic_ai.embeddings.bedrock.BedrockEmbeddingSettings]:
+
+- `bedrock_nova_truncate` — Fine-grained truncation control: `'NONE'` (default, error on overflow), `'START'`, or `'END'`. Overrides the base `truncate` setting.
+- `bedrock_nova_embedding_purpose` — By default, `embed_query()` uses `'GENERIC_RETRIEVAL'` and `embed_documents()` uses `'GENERIC_INDEX'`. Also accepts `'TEXT_RETRIEVAL'`, `'CLASSIFICATION'`, or `'CLUSTERING'`.
+
+```python {title="bedrock_nova.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingSettings
+
+embedder = Embedder(
+    'bedrock:amazon.nova-2-multimodal-embeddings-v1:0',
+    settings=BedrockEmbeddingSettings(
+        dimensions=1024,
+        bedrock_nova_embedding_purpose='TEXT_RETRIEVAL',
+        truncate=True,
+    ),
+)
+```
+
+#### Concurrency Settings
+
+Models that don't support batch embedding (Titan and Nova) make individual API requests for each input text. By default, these requests run concurrently with a maximum of 5 parallel requests.
+
+You can adjust this with the `bedrock_max_concurrency` setting:
+
+```python {title="bedrock_concurrency.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingSettings
+
+# Increase concurrency for faster throughput
+embedder = Embedder(
+    'bedrock:amazon.titan-embed-text-v2:0',
+    settings=BedrockEmbeddingSettings(bedrock_max_concurrency=10),
+)
+
+# Or reduce concurrency to avoid rate limits
+embedder = Embedder(
+    'bedrock:amazon.nova-2-multimodal-embeddings-v1:0',
+    settings=BedrockEmbeddingSettings(bedrock_max_concurrency=2),
+)
+```
+
+#### Regional Prefixes (Cross-Region Inference)
+
+Bedrock supports cross-region inference using geographic prefixes like `us.`, `eu.`, or `apac.`:
+
+```python {title="bedrock_regional.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('bedrock:us.amazon.titan-embed-text-v2:0')
+```
+
+#### Using AWS Application Inference Profiles
+
+Set [`bedrock_inference_profile`][pydantic_ai.embeddings.bedrock.BedrockEmbeddingSettings.bedrock_inference_profile] to route requests through an inference profile while keeping the base model name for detecting model capabilities:
+
+```python {title="bedrock_inference_profile.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingModel
+from pydantic_ai.providers.bedrock import BedrockProvider
+
+provider = BedrockProvider(region_name='us-east-1')
+
+model = BedrockEmbeddingModel(
+    'amazon.titan-embed-text-v2:0',
+    provider=provider,
+    settings={
+        'bedrock_inference_profile': 'arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-embed-profile',
+    },
+)
+embedder = Embedder(model)
+```
+
+#### Using a Custom Provider
+
+For advanced configuration like explicit credentials or a custom boto3 client, you can create a [`BedrockProvider`][pydantic_ai.providers.bedrock.BedrockProvider] directly. See the [Bedrock provider documentation](models/bedrock.md#provider-argument) for more details.
+
+```python {title="bedrock_provider.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.bedrock import BedrockEmbeddingModel
+from pydantic_ai.providers.bedrock import BedrockProvider
+
+provider = BedrockProvider(
+    region_name='us-west-2',
+    aws_access_key_id='your-access-key',
+    aws_secret_access_key='your-secret-key',
+)
+
+model = BedrockEmbeddingModel('amazon.titan-embed-text-v2:0', provider=provider)
+embedder = Embedder(model)
+```
+
+!!! note "Token Counting"
+    Bedrock embedding models do not support the `count_tokens()` method because AWS Bedrock's token counting API only works with text generation models (Claude, Llama, etc.), not embedding models. Calling `count_tokens()` will raise `NotImplementedError`.
+
+### Sentence Transformers (Local)
+
+[`SentenceTransformerEmbeddingModel`][pydantic_ai.embeddings.sentence_transformers.SentenceTransformerEmbeddingModel] runs embeddings locally using the [sentence-transformers](https://www.sbert.net/) library, giving you access to the thousands of [embedding models on Hugging Face](https://huggingface.co/models?library=sentence-transformers) without any API calls. This is ideal for:
+
+- **Privacy** — Data never leaves your infrastructure
+- **Cost** — No API charges for high-volume workloads
+- **Offline use** — No internet connection required after model download
+- **Specialized domains or languages** - Pick models trained for code, multilingual, biomedical, legal, etc. from the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard)
+
+#### Install
+
+To use Sentence Transformers embedding models, you need to install `pydantic-ai-slim` with the `sentence-transformers` optional group:
+
+```bash
+pip/uv-add "pydantic-ai-slim[sentence-transformers]"
+```
+
+#### Usage
+
+```python {title="sentence_transformers_embeddings.py" max_py="3.13"}
+from pydantic_ai import Embedder
+
+# Model is downloaded from Hugging Face on first use
+embedder = Embedder('sentence-transformers:lightonai/DenseOn')
+
+
+async def main():
+    result = await embedder.embed_query('Hello world')
+    print(len(result.embeddings[0]))
+    #> 768
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+[`lightonai/DenseOn`](https://huggingface.co/lightonai/DenseOn) is a strong recent 149M-parameter general-purpose model that encodes queries and documents asymmetrically: [`embed_query()`][pydantic_ai.embeddings.Embedder.embed_query] and [`embed_documents()`][pydantic_ai.embeddings.Embedder.embed_documents] automatically apply the model's `query:` / `document:` prompts. See the [Sentence Transformers pretrained models](https://www.sbert.net/docs/sentence_transformer/pretrained_models.html) documentation and the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard) for more options; see also [Choosing a model](#choosing-a-model) above.
+
+#### Device Selection
+
+Control which device to use for inference:
+
+```python {title="sentence_transformers_device.py" max_py="3.13"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.sentence_transformers import (
+    SentenceTransformersEmbeddingSettings,
+)
+
+embedder = Embedder(
+    'sentence-transformers:sentence-transformers/all-MiniLM-L6-v2',
+    settings=SentenceTransformersEmbeddingSettings(
+        sentence_transformers_device='cuda',  # Use GPU
+        sentence_transformers_normalize_embeddings=True,  # L2 normalize
+    ),
+)
+```
+
+#### Using an Existing Model Instance
+
+If you need more control over model initialization:
+
+```python {title="sentence_transformers_instance.py" max_py="3.13"}
+from sentence_transformers import SentenceTransformer
+
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings.sentence_transformers import (
+    SentenceTransformerEmbeddingModel,
+)
+
+# Create and configure the model yourself
+st_model = SentenceTransformer('microsoft/harrier-oss-v1-270m', device='cpu')
+
+# Wrap it for use with Pydantic AI
+model = SentenceTransformerEmbeddingModel(st_model)
+embedder = Embedder(model)
+```
+
+## Settings
+
+[`EmbeddingSettings`][pydantic_ai.embeddings.EmbeddingSettings] provides common configuration options that work across providers:
+
+- `dimensions`: Reduce the output embedding dimensions (supported by OpenAI, Google, Cohere, Bedrock, VoyageAI)
+- `truncate`: When `True`, truncate input text that exceeds the model's context length instead of raising an error (supported by Cohere, Bedrock, VoyageAI)
+
+Settings can be specified at the embedder level (applied to all calls) or per-call:
+
+```python {title="embedding_settings.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings import EmbeddingSettings
+
+# Default settings for all calls
+embedder = Embedder(
+    'openai:text-embedding-3-small',
+    settings=EmbeddingSettings(dimensions=512),
+)
+
+
+async def main():
+    # Override for a specific call
+    result = await embedder.embed_query(
+        'Hello world',
+        settings=EmbeddingSettings(dimensions=256),
+    )
+    print(len(result.embeddings[0]))
+    #> 256
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Token Counting
+
+You can check token counts before embedding to avoid exceeding model limits:
+
+```python {title="token_counting.py"}
+from pydantic_ai import Embedder
+
+embedder = Embedder('openai:text-embedding-3-small')
+
+
+async def main():
+    text = 'Hello world, this is a test.'
+
+    # Count tokens in text
+    token_count = await embedder.count_tokens(text)
+    print(f'Tokens: {token_count}')
+    #> Tokens: 7
+
+    # Check model's maximum input tokens (returns None if unknown)
+    max_tokens = await embedder.max_input_tokens()
+    print(f'Max tokens: {max_tokens}')
+    #> Max tokens: 1024
+```
+
+_(This example is complete, it can be run "as is" — you'll need to add `asyncio.run(main())` to run `main`)_
+
+## Testing
+
+Use [`TestEmbeddingModel`][pydantic_ai.embeddings.TestEmbeddingModel] for testing without making API calls:
+
+```python {title="testing_embeddings.py"}
+from pydantic_ai import Embedder
+from pydantic_ai.embeddings import TestEmbeddingModel
+
+
+async def test_my_rag_system():
+    embedder = Embedder('openai:text-embedding-3-small')
+    test_model = TestEmbeddingModel()
+
+    with embedder.override(model=test_model):
+        result = await embedder.embed_query('test query')
+
+        # TestEmbeddingModel returns deterministic embeddings
+        assert result.embeddings[0] == [1.0] * 8
+
+        # Check what settings were used
+        assert test_model.last_settings is not None
+```
+
+## Instrumentation
+
+Enable OpenTelemetry instrumentation for debugging and monitoring:
+
+```python {title="instrumented_embeddings.py"}
+import logfire
+
+from pydantic_ai import Embedder
+
+logfire.configure()
+
+# Instrument a specific embedder
+embedder = Embedder('openai:text-embedding-3-small', instrument=True)
+
+# Or instrument all embedders globally
+Embedder.instrument_all()
+```
+
+See the [Debugging and Monitoring guide](logfire.md) for more details on using Logfire with Pydantic AI.
+
+## Two-stage retrieval with rerankers
+
+For high-quality retrieval, a common pattern is **two-stage**: first use an embedding model to pull a broad shortlist of candidates cheaply, then use a **cross-encoder reranker** to score each candidate against the query more precisely. The cross-encoder reads the query and document *together*, so it's slower than an embedding lookup but dramatically more accurate, making it ideal for narrowing a top-100 recall list down to the top-5 results you actually hand to the LLM.
+
+Pydantic AI does not ship a reranker provider class, so you bring your own. The most common local option is a `CrossEncoder` from `sentence-transformers`:
+
+```python {title="rerank.py" max_py="3.13"}
+import asyncio
+from functools import cache
+
+from sentence_transformers import CrossEncoder
+
+
+@cache
+def get_reranker() -> CrossEncoder:
+    # Loaded lazily on first call, then reused.
+    return CrossEncoder('cross-encoder/ms-marco-MiniLM-L6-v2')
+
+
+async def rerank(query: str, candidates: list[str], top_k: int = 3) -> list[str]:
+    """Rerank retrieval candidates by relevance to `query`."""
+    reranker = get_reranker()
+    # CrossEncoder.rank is blocking, so run it off the event loop.
+    ranked = await asyncio.to_thread(
+        reranker.rank, query, candidates, top_k=top_k, return_documents=True
+    )
+    return [item['text'] for item in ranked]
+```
+
+Call `rerank()` on the candidates returned by your vector search (for example, in the `retrieve` tool of the [RAG example](examples/rag.md)) before handing the results to the LLM.
+
+!!! tip "Managed reranker alternatives"
+    If you'd rather not run a reranker locally, several providers offer hosted rerankers, including [Cohere Rerank](https://docs.cohere.com/docs/rerank-overview), [VoyageAI Rerank](https://docs.voyageai.com/docs/reranker), and [Jina Rerank](https://jina.ai/reranker). Call their HTTP clients or SDKs from a helper function with the same shape as `rerank()` above.
+
+## Building Custom Embedding Models
+
+To integrate a custom embedding provider, subclass [`EmbeddingModel`][pydantic_ai.embeddings.EmbeddingModel]:
+
+```python {title="custom_embedding_model.py"}
+from collections.abc import Sequence
+
+from pydantic_ai.embeddings import EmbeddingModel, EmbeddingResult, EmbeddingSettings
+from pydantic_ai.embeddings.result import EmbedInputType
+
+
+class MyCustomEmbeddingModel(EmbeddingModel):
+    @property
+    def model_name(self) -> str:
+        return 'my-custom-model'
+
+    @property
+    def system(self) -> str:
+        return 'my-provider'
+
+    async def embed(
+        self,
+        inputs: str | Sequence[str],
+        *,
+        input_type: EmbedInputType,
+        settings: EmbeddingSettings | None = None,
+    ) -> EmbeddingResult:
+        inputs, settings = self.prepare_embed(inputs, settings)
+
+        # Call your embedding API here
+        embeddings = [[0.1, 0.2, 0.3] for _ in inputs]  # Placeholder
+
+        return EmbeddingResult(
+            embeddings=embeddings,
+            inputs=inputs,
+            input_type=input_type,
+            model_name=self.model_name,
+            provider_name=self.system,
+        )
+```
+
+Use [`WrapperEmbeddingModel`][pydantic_ai.embeddings.WrapperEmbeddingModel] if you want to wrap an existing model to add custom behavior like caching or logging.

docs/evals.md

@@ -0,0 +1,286 @@
+---
+title: Pydantic Evals
+---
+
+# Pydantic Evals
+
+**Pydantic Evals** is a powerful evaluation framework for systematically testing and evaluating AI systems, from simple LLM calls to complex multi-agent applications.
+
+## Design Philosophy
+
+!!! note "Code-First Approach"
+    Pydantic Evals follows a code-first philosophy where all evaluation components are defined in Python. This differs from platforms with web-based configuration. You write and run evals in code, and can write the results to disk or view them in your terminal or in [Pydantic Logfire](https://logfire.pydantic.dev/docs/guides/web-ui/evals/).
+
+!!! danger "Evals are an Emerging Practice"
+    Unlike unit tests, evals are an emerging art/science. Anyone who claims to know exactly how your evals should be defined can safely be ignored. We've designed Pydantic Evals to be flexible and useful without being too opinionated.
+
+
+## Quick Navigation
+
+**Getting Started:**
+
+- [Installation](#installation)
+- [Quick Start](evals/quick-start.md)
+- [Core Concepts](evals/core-concepts.md)
+
+**Evaluators:**
+
+- [Evaluators Overview](evals/evaluators/overview.md) - Compare evaluator types and learn when to use each approach
+- [Built-in Evaluators](evals/evaluators/built-in.md) - Complete reference for exact match, instance checks, and other ready-to-use evaluators
+- [LLM as a Judge](evals/evaluators/llm-judge.md) - Use LLMs to evaluate subjective qualities, complex criteria, and natural language outputs
+- [Custom Evaluators](evals/evaluators/custom.md) - Implement domain-specific scoring logic and custom evaluation metrics
+- [Span-Based Evaluation](evals/evaluators/span-based.md) - Evaluate internal agent behavior (tool calls, execution flow) using OpenTelemetry traces. Essential for complex agents where correctness depends on _how_ the answer was reached, not just the final output. Also ensures eval assertions align with production telemetry.
+
+**How-To Guides:**
+
+- [Logfire Integration](evals/how-to/logfire-integration.md) - Visualize results
+- [Dataset Management](evals/how-to/dataset-management.md) - Save, load, generate
+- [Concurrency & Performance](evals/how-to/concurrency.md) - Control parallel execution
+- [Retry Strategies](evals/how-to/retry-strategies.md) - Handle transient failures
+- [Metrics & Attributes](evals/how-to/metrics-attributes.md) - Track custom data
+- [Case Lifecycle Hooks](evals/how-to/lifecycle.md) - Per-case setup, teardown, and context enrichment
+
+**Examples:**
+
+- [Simple Validation](evals/examples/simple-validation.md) - Basic example
+
+**Reference:**
+
+- [API Documentation](api/pydantic_evals/dataset.md)
+
+## Code-First Evaluation
+
+Pydantic Evals follows a **code-first approach** where you define all evaluation components (datasets, experiments, tasks, cases and evaluators) in Python code, or as serialized data loaded by Python code. This differs from platforms with fully web-based configuration.
+
+When you run an _Experiment_ you'll see a progress indicator and can print the results wherever you run your python code (IDE, terminal, etc). You also get a report object back that you can serialize and store or send to a notebook or other application for further visualization and analysis.
+
+If you are using [Pydantic Logfire](https://logfire.pydantic.dev/docs/guides/web-ui/evals/), your experiment results automatically appear in the Logfire web interface for visualization, comparison, and collaborative analysis. Logfire serves as a observability layer - you write and run evals in code, then view and analyze results in the web UI.
+
+## Installation
+
+To install the Pydantic Evals package, run:
+
+```bash
+pip/uv-add pydantic-evals
+```
+
+`pydantic-evals` does not depend on `pydantic-ai`, but has an optional dependency on `logfire` if you'd like to
+use OpenTelemetry traces in your evals, or send evaluation results to [logfire](https://pydantic.dev/logfire).
+
+```bash
+pip/uv-add 'pydantic-evals[logfire]'
+```
+
+## Pydantic Evals Data Model
+
+Pydantic Evals is built around a simple data model:
+
+### Data Model Diagram
+
+```
+Dataset (1) ──────────── (Many) Case
+│                        │
+│                        │
+└─── (Many) Experiment ──┴─── (Many) Case results
+     │
+     └─── (1) Task
+     │
+     └─── (Many) Evaluator
+```
+
+### Key Relationships
+
+1. **Dataset → Cases**: One Dataset contains many Cases
+2. **Dataset → Experiments**: One Dataset can be used across many Experiments over time
+3. **Experiment → Case results**: One Experiment generates results by executing each Case
+4. **Experiment → Task**: One Experiment evaluates one defined Task
+5. **Experiment → Evaluators**: One Experiment uses multiple Evaluators. Dataset-wide Evaluators are run against all Cases, and Case-specific Evaluators against their respective Cases
+
+### Data Flow
+
+1. **Dataset creation**: Define cases and evaluators in YAML/JSON, or directly in Python
+2. **Experiment execution**: Run `dataset.evaluate_sync(task_function)`
+3. **Cases run**: Each Case is executed against the Task
+4. **Evaluation**: Evaluators score the Task outputs for each Case
+5. **Results**: All Case results are collected into a summary report
+
+!!! note "A metaphor"
+
+    A useful metaphor (although not perfect) is to think of evals like a **Unit Testing** framework:
+
+    - **Cases + Evaluators** are your individual unit tests - each one
+    defines a specific scenario you want to test, complete with inputs
+    and expected outcomes. Just like a unit test, a case asks: _"Given
+    this input, does my system produce the right output?"_
+
+    -  **Datasets** are like test suites - they are the scaffolding that holds your unit
+    tests together. They group related cases and define shared
+    evaluation criteria that should apply across all tests in the suite.
+
+    - **Experiments** are like running your entire test suite and getting a
+    report. When you execute `dataset.evaluate_sync(my_ai_function)`,
+    you're running all your cases against your AI system and
+    collecting the results - just like running `pytest` and getting a
+    summary of passes, failures, and performance metrics.
+
+    The key difference from traditional unit testing is that AI systems are
+    probabilistic. If you're type checking you'll still get a simple pass/fail,
+    but scores for text outputs are likely qualitative and/or categorical,
+    and more open to interpretation.
+
+For a deeper understanding, see [Core Concepts](evals/core-concepts.md).
+
+## Datasets and Cases
+
+In Pydantic Evals, everything begins with [`Dataset`][pydantic_evals.dataset.Dataset]s and [`Case`][pydantic_evals.dataset.Case]s:
+
+- **[`Dataset`][pydantic_evals.dataset.Dataset]**: A collection of test Cases designed for the evaluation of a specific task or function
+- **[`Case`][pydantic_evals.dataset.Case]**: A single test scenario corresponding to Task inputs, with optional expected outputs, metadata, and case-specific evaluators
+
+```python {title="simple_eval_dataset.py"}
+from pydantic_evals import Case, Dataset
+
+case1 = Case(
+    name='simple_case',
+    inputs='What is the capital of France?',
+    expected_output='Paris',
+    metadata={'difficulty': 'easy'},
+)
+
+dataset = Dataset(name='capital_quiz', cases=[case1])
+```
+
+_(This example is complete, it can be run "as is")_
+
+See [Dataset Management](evals/how-to/dataset-management.md) to learn about saving, loading, and generating datasets.
+
+## Evaluators
+
+[`Evaluator`][pydantic_evals.evaluators.Evaluator]s analyze and score the results of your Task when tested against a Case.
+
+These can be deterministic, code-based checks (such as testing model output format with a regex, or checking for the appearance of PII or sensitive data), or they can assess non-deterministic model outputs for qualities like accuracy, precision/recall, hallucinations, or instruction-following.
+
+While both kinds of testing are useful in LLM systems, classical code-based tests are cheaper and easier than tests which require either human or machine review of model outputs.
+
+Pydantic Evals includes several [built-in evaluators](evals/evaluators/built-in.md) and allows you to define [custom evaluators](evals/evaluators/custom.md):
+
+```python {title="simple_eval_evaluator.py" requires="simple_eval_dataset.py"}
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+from pydantic_evals.evaluators.common import IsInstance
+
+from simple_eval_dataset import dataset
+
+dataset.add_evaluator(IsInstance(type_name='str'))  # (1)!
+
+
+@dataclass
+class MyEvaluator(Evaluator):
+    async def evaluate(self, ctx: EvaluatorContext[str, str]) -> float:  # (2)!
+        if ctx.output == ctx.expected_output:
+            return 1.0
+        elif (
+            isinstance(ctx.output, str)
+            and ctx.expected_output.lower() in ctx.output.lower()
+        ):
+            return 0.8
+        else:
+            return 0.0
+
+
+dataset.add_evaluator(MyEvaluator())
+```
+
+1. You can add built-in evaluators to a dataset using the [`add_evaluator`][pydantic_evals.dataset.Dataset.add_evaluator] method.
+2. This custom evaluator returns a simple score based on whether the output matches the expected output.
+
+_(This example is complete, it can be run "as is")_
+
+Learn more:
+
+- [Evaluators Overview](evals/evaluators/overview.md) - When to use different types
+- [Built-in Evaluators](evals/evaluators/built-in.md) - Complete reference
+- [LLM Judge](evals/evaluators/llm-judge.md) - Using LLMs as evaluators
+- [Custom Evaluators](evals/evaluators/custom.md) - Write your own logic
+- [Span-Based Evaluation](evals/evaluators/span-based.md) - Analyze execution traces
+
+## Running Experiments
+
+Performing evaluations involves running a task against all cases in a dataset, also known as running an "experiment".
+
+Putting the above two examples together and using the more declarative `evaluators` kwarg to [`Dataset`][pydantic_evals.dataset.Dataset]:
+
+```python {title="simple_eval_complete.py"}
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext, IsInstance
+
+case1 = Case(  # (1)!
+    name='simple_case',
+    inputs='What is the capital of France?',
+    expected_output='Paris',
+    metadata={'difficulty': 'easy'},
+)
+
+
+class MyEvaluator(Evaluator[str, str]):
+    def evaluate(self, ctx: EvaluatorContext[str, str]) -> float:
+        if ctx.output == ctx.expected_output:
+            return 1.0
+        elif (
+            isinstance(ctx.output, str)
+            and ctx.expected_output.lower() in ctx.output.lower()
+        ):
+            return 0.8
+        else:
+            return 0.0
+
+
+dataset = Dataset(
+    name='capital_quiz',
+    cases=[case1],
+    evaluators=[IsInstance(type_name='str'), MyEvaluator()],  # (2)!
+)
+
+
+async def guess_city(question: str) -> str:  # (3)!
+    return 'Paris'
+
+
+report = dataset.evaluate_sync(guess_city)  # (4)!
+report.print(include_input=True, include_output=True, include_durations=False)  # (5)!
+"""
+                              Evaluation Summary: guess_city
+┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
+┃ Case ID     ┃ Inputs                         ┃ Outputs ┃ Scores            ┃ Assertions ┃
+┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
+│ simple_case │ What is the capital of France? │ Paris   │ MyEvaluator: 1.00 │ ✔          │
+├─────────────┼────────────────────────────────┼─────────┼───────────────────┼────────────┤
+│ Averages    │                                │         │ MyEvaluator: 1.00 │ 100.0% ✔   │
+└─────────────┴────────────────────────────────┴─────────┴───────────────────┴────────────┘
+"""
+```
+
+1. Create a [test case][pydantic_evals.dataset.Case] as above
+2. Create a [`Dataset`][pydantic_evals.dataset.Dataset] with test cases and [`evaluators`][pydantic_evals.dataset.Dataset.evaluators]
+3. Our function to evaluate.
+4. Run the evaluation with [`evaluate_sync`][pydantic_evals.dataset.Dataset.evaluate_sync], which runs the function against all test cases in the dataset, and returns an [`EvaluationReport`][pydantic_evals.reporting.EvaluationReport] object.
+5. Print the report with [`print`][pydantic_evals.reporting.EvaluationReport.print], which shows the results of the evaluation. We have omitted duration here just to keep the printed output from changing from run to run.
+
+_(This example is complete, it can be run "as is")_
+
+See [Quick Start](evals/quick-start.md) for more examples and [Concurrency & Performance](evals/how-to/concurrency.md) to learn about controlling parallel execution.
+
+## API Reference
+
+For comprehensive coverage of all classes, methods, and configuration options, see the detailed [API Reference documentation](https://ai.pydantic.dev/api/pydantic_evals/dataset/).
+
+## Next Steps
+
+<!-- TODO - this would be the perfect place for a full tutorial or case study  -->
+1. **Start with simple evaluations** using [Quick Start](evals/quick-start.md)
+2. **Understand the data model** with [Core Concepts](evals/core-concepts.md)
+3. **Explore built-in evaluators** in [Built-in Evaluators](evals/evaluators/built-in.md)
+4. **Integrate with Logfire** for visualization: [Logfire Integration](evals/how-to/logfire-integration.md)
+5. **Build comprehensive test suites** with [Dataset Management](evals/how-to/dataset-management.md)
+6. **Implement custom evaluators** for domain-specific metrics: [Custom Evaluators](evals/evaluators/custom.md)

docs/evals/core-concepts.md

@@ -0,0 +1,503 @@
+# Core Concepts
+
+This page explains the key concepts in Pydantic Evals and how they work together.
+
+## Overview
+
+Pydantic Evals is built around these core concepts:
+
+- **[`Dataset`][pydantic_evals.dataset.Dataset]** - A static definition containing test cases and evaluators
+- **[`Case`][pydantic_evals.dataset.Case]** - A single test scenario with inputs and optional expected outputs
+- **[`Evaluator`][pydantic_evals.evaluators.Evaluator]** - Logic for scoring or validating individual outputs
+- **[`ReportEvaluator`][pydantic_evals.evaluators.ReportEvaluator]** - Logic for analyzing full experiment results (e.g., confusion matrices, accuracy)
+- **Experiment** - The act of running a task function against all cases in a dataset. (This corresponds to a call to `Dataset.evaluate`.)
+- **[`EvaluationReport`][pydantic_evals.reporting.EvaluationReport]** - The results from running an experiment
+
+The key distinction is between:
+
+- **Definition** (`Dataset` with `Case`s, `Evaluator`s, and `ReportEvaluator`s) - what you want to test
+- **Execution** (Experiment) - running your task against those tests
+- **Results** (`EvaluationReport` with case results and experiment-wide analyses) - what happened during the experiment
+
+## Unit Testing Analogy
+
+A helpful way to think about Pydantic Evals:
+
+| Unit Testing | Pydantic Evals |
+|--------------|----------------|
+| Test function | [`Case`][pydantic_evals.dataset.Case] + [`Evaluator`][pydantic_evals.evaluators.Evaluator] |
+| Test suite | [`Dataset`][pydantic_evals.dataset.Dataset] |
+| Running tests (`pytest`) | **Experiment** (`dataset.evaluate(task)`) |
+| Test report | [`EvaluationReport`][pydantic_evals.reporting.EvaluationReport] |
+| `assert` | Evaluator returning `bool` |
+
+**Key Difference**: AI systems are probabilistic, so instead of simple pass/fail, evaluations can have:
+
+- Quantitative scores (0.0 to 1.0)
+- Qualitative labels ("good", "acceptable", "poor")
+- Pass/fail assertions with explanatory reasons
+
+Just like you can run `pytest` multiple times on the same test suite, you can run multiple experiments on the same dataset to compare different implementations or track changes over time.
+
+## Dataset
+
+A [`Dataset`][pydantic_evals.dataset.Dataset] is a collection of test cases and evaluators that define an evaluation suite.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import IsInstance
+
+dataset = Dataset(
+    name='my_eval_suite',
+    cases=[
+        Case(inputs='test input', expected_output='test output'),
+    ],
+    evaluators=[
+        IsInstance(type_name='str'),
+    ],
+)
+```
+
+### Key Features
+
+- **Type-safe**: Generic over `InputsT`, `OutputT`, and `MetadataT` types
+- **Serializable**: Can be saved to/loaded from YAML or JSON files
+- **Evaluable**: Run against any function with matching input/output types
+
+### `Dataset`-Level vs `Case`-Level Evaluators
+
+Evaluators can be defined at two levels:
+
+- **`Dataset`-level**: Apply to all cases in the dataset
+- **`Case`-level**: Apply only to specific cases
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import EqualsExpected, IsInstance
+
+dataset = Dataset(
+    name='case_level_evaluators',
+    cases=[
+        Case(
+            name='special_case',
+            inputs='test',
+            expected_output='TEST',
+            evaluators=[
+                # This evaluator only runs for this case
+                EqualsExpected(),
+            ],
+        ),
+    ],
+    evaluators=[
+        # This evaluator runs for ALL cases
+        IsInstance(type_name='str'),
+    ],
+)
+```
+
+## Experiments
+
+An **Experiment** is what happens when you execute a task function against all cases in a dataset. This is the bridge between your static test definition (the Dataset) and your results (the EvaluationReport).
+
+### Running an Experiment
+
+You run an experiment by calling [`evaluate()`][pydantic_evals.dataset.Dataset.evaluate] or [`evaluate_sync()`][pydantic_evals.dataset.Dataset.evaluate_sync] on a dataset:
+
+```python
+from pydantic_evals import Case, Dataset
+
+# Define your dataset (static definition)
+dataset = Dataset(
+    name='uppercase_experiment',
+    cases=[
+        Case(inputs='hello', expected_output='HELLO'),
+        Case(inputs='world', expected_output='WORLD'),
+    ],
+)
+
+# Define your task
+def uppercase_task(text: str) -> str:
+    return text.upper()
+
+# Run the experiment (execution)
+report = dataset.evaluate_sync(uppercase_task)
+```
+
+### What Happens During an Experiment
+
+When you run an experiment:
+
+1. **Setup**: The dataset loads all cases, evaluators, and report evaluators
+2. **Execution**: For each case:
+    1. The task function is called with `case.inputs`
+    2. Execution time is measured and OpenTelemetry spans are captured (if `logfire` is configured)
+    3. The outputs of the task function for each case are recorded
+3. **Case Evaluation**: For each case output:
+    1. All dataset-level evaluators are run
+    2. Case-specific evaluators are run (if any)
+    3. Results are collected (scores, assertions, labels)
+4. **Report Evaluation**: If report evaluators are configured, they run over the full set of results to produce experiment-wide analyses (confusion matrices, precision-recall curves, scalar metrics, tables, etc.)
+5. **Reporting**: All results are aggregated into an [`EvaluationReport`][pydantic_evals.reporting.EvaluationReport], including both per-case results and experiment-wide analyses
+
+### Multiple Experiments from One Dataset
+
+A key feature of Pydantic Evals is that you can run the same dataset against different task implementations:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import EqualsExpected
+
+dataset = Dataset(
+    name='comparison_test',
+    cases=[
+        Case(inputs='hello', expected_output='HELLO'),
+    ],
+    evaluators=[EqualsExpected()],
+)
+
+
+# Original implementation
+def task_v1(text: str) -> str:
+    return text.upper()
+
+
+# Improved implementation (with exclamation)
+def task_v2(text: str) -> str:
+    return text.upper() + '!'
+
+
+# Compare results
+report_v1 = dataset.evaluate_sync(task_v1)
+report_v2 = dataset.evaluate_sync(task_v2)
+
+avg_v1 = report_v1.averages()
+avg_v2 = report_v2.averages()
+print(f'V1 pass rate: {avg_v1.assertions if avg_v1 and avg_v1.assertions else 0}')
+#> V1 pass rate: 1.0
+print(f'V2 pass rate: {avg_v2.assertions if avg_v2 and avg_v2.assertions else 0}')
+#> V2 pass rate: 0
+```
+
+This allows you to:
+
+- **Compare implementations** across versions
+- **Track performance** over time
+- **A/B test** different approaches
+- **Validate changes** before deployment
+
+## Case
+
+A [`Case`][pydantic_evals.dataset.Case] represents a single test scenario with specific inputs and optional expected outputs.
+
+```python
+from pydantic_evals import Case
+from pydantic_evals.evaluators import EqualsExpected
+
+case = Case(
+    name='test_uppercase',  # Optional, but recommended for reporting
+    inputs='hello world',  # Required: inputs to your task
+    expected_output='HELLO WORLD',  # Optional: expected output
+    metadata={'category': 'basic'},  # Optional: arbitrary metadata
+    evaluators=[EqualsExpected()],  # Optional: case-specific evaluators
+)
+```
+
+### Case Components
+
+#### Inputs
+The inputs to pass to the task being evaluated. Can be any type:
+
+```python
+from pydantic import BaseModel
+
+from pydantic_evals import Case
+
+
+class MyInputModel(BaseModel):
+    field1: str
+
+
+# Simple types
+Case(inputs='hello')
+Case(inputs=42)
+
+# Complex types
+Case(inputs={'query': 'What is AI?', 'max_tokens': 100})
+Case(inputs=MyInputModel(field1='value'))
+```
+
+#### Expected Output
+The expected result, used by evaluators like [`EqualsExpected`][pydantic_evals.evaluators.EqualsExpected]:
+
+```python
+from pydantic_evals import Case
+
+Case(
+    inputs='2 + 2',
+    expected_output='4',
+)
+```
+
+If no `expected_output` is provided, evaluators that require it (like `EqualsExpected`) will skip that case.
+
+#### Metadata
+Arbitrary data that evaluators can access via [`EvaluatorContext`][pydantic_evals.evaluators.EvaluatorContext]:
+
+```python
+from pydantic_evals import Case
+
+Case(
+    inputs='question',
+    metadata={
+        'difficulty': 'hard',
+        'category': 'math',
+        'source': 'exam_2024',
+    },
+)
+```
+
+Metadata is useful for:
+
+- Filtering cases during analysis
+- Providing context to evaluators
+- Organizing test suites
+
+#### Evaluators
+
+Cases can have their own evaluators that only run for that specific case. This is particularly powerful for building comprehensive evaluation suites where different cases have different requirements - if you could write one evaluator rubric that worked perfectly for all cases, you'd just incorporate it into your agent instructions. Case-specific [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] evaluators are especially useful for quickly building maintainable golden datasets by describing what "good" looks like for each scenario. See [Case-specific evaluators](evaluators/overview.md#case-specific-evaluators) for a more detailed explanation and examples.
+
+## Evaluator
+
+An [`Evaluator`][pydantic_evals.evaluators.Evaluator] assesses the output of your task and returns one or more scores, labels, or assertions. Each score, label or assertion can also have an optional string-value reason associated.
+
+### Evaluator Types
+
+Evaluators return different types of results:
+
+| Return Type | Purpose | Example |
+|-------------|---------|---------|
+| `bool` | **Assertion** - Pass/fail check | `True` → ✔, `False` → ✗ |
+| `int` or `float` | **Score** - Numeric quality metric | `0.95`, `87` |
+| `str` | **Label** - Categorical result | `"correct"`, `"hallucination"` |
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ExactMatch(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return ctx.output == ctx.expected_output  # Assertion
+
+
+@dataclass
+class Confidence(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> float:
+        # Analyze output and return confidence score
+        return 0.95  # Score
+
+
+@dataclass
+class Classifier(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> str:
+        if 'error' in ctx.output.lower():
+            return 'error'  # Label
+        return 'success'
+```
+
+Evaluators can also return instances of [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason], and dictionaries mapping labels to output values.
+See the [custom evaluator return types](evaluators/custom.md#return-types) docs for more detail.
+
+### EvaluatorContext
+
+All evaluators receive an [`EvaluatorContext`][pydantic_evals.evaluators.EvaluatorContext] containing:
+
+- `name`: Case name (optional)
+- `inputs`: Task inputs
+- `metadata`: Case metadata (optional)
+- `expected_output`: Expected output (optional)
+- `output`: Actual output from task
+- `duration`: Task execution time in seconds
+- `span_tree`: OpenTelemetry spans (if `logfire` is configured)
+- `attributes`: Custom attributes dict
+- `metrics`: Custom metrics dict
+
+### Multiple Evaluations
+
+Evaluators can return multiple results by returning a dictionary:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class MultiCheck(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool | float | str]:
+        return {
+            'is_valid': isinstance(ctx.output, str),  # Assertion
+            'length': len(ctx.output),  # Metric
+            'category': 'long' if len(ctx.output) > 100 else 'short',  # Label
+        }
+```
+
+### Evaluation Reasons
+
+Add explanations to your evaluations using [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason]:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class SmartCheck(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        if ctx.output == ctx.expected_output:
+            return EvaluationReason(
+                value=True,
+                reason='Exact match with expected output',
+            )
+        return EvaluationReason(
+            value=False,
+            reason=f'Expected {ctx.expected_output!r}, got {ctx.output!r}',
+        )
+```
+
+Reasons appear in reports when using `include_reasons=True`.
+
+## Evaluation Report
+
+An [`EvaluationReport`][pydantic_evals.reporting.EvaluationReport] is the result of running an experiment. It contains all the data from executing your task against the dataset's cases and running all evaluators.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import EqualsExpected
+
+dataset = Dataset(
+    name='report_example',
+    cases=[Case(inputs='hello', expected_output='HELLO')],
+    evaluators=[EqualsExpected()],
+)
+
+
+def my_task(text: str) -> str:
+    return text.upper()
+
+
+# Run an experiment
+report = dataset.evaluate_sync(my_task)
+
+# Print to console
+report.print()
+"""
+    Evaluation Summary: my_task
+┏━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━┓
+┃ Case ID  ┃ Assertions ┃ Duration ┃
+┡━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━┩
+│ Case 1   │ ✔          │     10ms │
+├──────────┼────────────┼──────────┤
+│ Averages │ 100.0% ✔   │     10ms │
+└──────────┴────────────┴──────────┘
+"""
+
+# Access data programmatically
+for case in report.cases:
+    print(f'{case.name}: {case.scores}')
+    #> Case 1: {}
+```
+
+### Report Structure
+
+The [`EvaluationReport`][pydantic_evals.reporting.EvaluationReport] contains:
+
+- `name`: Experiment name
+- `cases`: List of successful case evaluations
+- `failures`: List of failed executions
+- `analyses`: List of experiment-wide analyses from [report evaluators](evaluators/report-evaluators.md) (confusion matrices, PR curves, scalars, tables)
+- `trace_id`: OpenTelemetry trace ID (optional)
+- `span_id`: OpenTelemetry span ID (optional)
+
+### ReportCase
+
+Each successfulcase result contains:
+
+**Case data:**
+
+- `name`: Case name
+- `inputs`: Task inputs
+- `metadata`: Case metadata (optional)
+- `expected_output`: Expected output (optional)
+- `output`: Actual output from task
+
+**Evaluation results:**
+
+- `scores`: Dictionary of numeric scores from evaluators
+- `labels`: Dictionary of categorical labels from evaluators
+- `assertions`: Dictionary of pass/fail assertions from evaluators
+
+**Performance data:**
+
+- `task_duration`: Task execution time
+- `total_duration`: Total time including evaluators
+
+**Additional data:**
+
+- `metrics`: Custom metrics dict
+- `attributes`: Custom attributes dict
+
+**Tracing:**
+
+- `trace_id`: OpenTelemetry trace ID (optional)
+- `span_id`: OpenTelemetry span ID (optional)
+
+**Errors:**
+
+- `evaluator_failures`: List of evaluator errors
+
+## Data Model Relationships
+
+Here's how the core concepts relate to each other:
+
+### Static Definition
+
+- A **Dataset** contains:
+  - Many **Cases** (test scenarios with inputs and expected outputs)
+  - Many **Evaluators** (logic for scoring individual outputs)
+  - Many **Report Evaluators** (logic for analyzing full experiment results)
+
+### Execution (Experiment)
+
+When you call `dataset.evaluate(task)`, an **Experiment** runs:
+
+- The **Task** function is executed against all **Cases** in the **Dataset**
+- All **Evaluators** are run (both dataset-level and case-specific) against each output as appropriate
+- One **EvaluationReport** is produced as the final output
+
+### Results
+
+- An **EvaluationReport** contains:
+  - Results for each **Case** (inputs, outputs, scores, assertions, labels)
+  - Experiment-wide **Analyses** from report evaluators (confusion matrices, PR curves, scalars, tables)
+  - Summary statistics (averages, pass rates)
+  - Performance data (durations)
+  - Tracing information (OpenTelemetry spans)
+
+### Key Relationships
+
+- **One Dataset → Many Experiments**: You can run the same dataset against different task implementations or multiple times to track changes
+- **One Experiment → One Report**: Each time you call `dataset.evaluate(...)`, you get one report
+- **One Experiment → Many Case Results**: The report contains results for every case in the dataset
+
+
+## Next Steps
+
+- **[Evaluators Overview](evaluators/overview.md)** - When to use different evaluator types
+- **[Native Evaluators](evaluators/built-in.md)** - Complete reference of provided evaluators
+- **[Custom Evaluators](evaluators/custom.md)** - Write your own evaluation logic
+- **[Report Evaluators](evaluators/report-evaluators.md)** - Experiment-wide analyses (confusion matrices, PR curves, etc.)
+- **[Dataset Management](how-to/dataset-management.md)** - Save, load, and generate datasets

docs/evals/evaluators/agentic.md

@@ -0,0 +1,333 @@
+# Agentic Evaluators
+
+Deterministic, span-based evaluators that grade an agent's *trajectory* — the sequence and arguments of tool calls — rather than just its final output.
+
+!!! note "Requires Logfire"
+    These evaluators read from the OpenTelemetry span tree captured during
+    task execution, so [`logfire`](../how-to/logfire-integration.md) must be
+    installed and configured:
+    ```bash
+    pip install 'pydantic-evals[logfire]'
+    ```
+    If spans aren't available, each evaluator returns a failing result
+    (`False` for the boolean evaluators, `0.0` for `TrajectoryMatch`) with
+    a reason pointing at logfire configuration, rather than raising.
+
+!!! warning "Locally-executed tools only"
+    These evaluators see tools whose execution produces a local OpenTelemetry
+    span — i.e. tools that Pydantic AI invokes itself. Provider-native or
+    server-side builtin tools (such as OpenAI's file search or Anthropic's
+    web search) don't produce local spans and are therefore invisible to
+    these evaluators. Use [`HasMatchingSpan`][pydantic_evals.evaluators.HasMatchingSpan]
+    against the provider's own spans, or the model's output, to assess those.
+
+!!! note "What counts as a tool call"
+    Every execution *attempt* produces a span, discriminated as follows:
+
+    - An attempt that ended in an error — the tool body raised an exception,
+      or requested a retry via `ModelRetry` — is **not** counted by default;
+      pass `include_failed=True` to count every attempt. The exception:
+      [`MaxToolCalls`][pydantic_evals.evaluators.MaxToolCalls] counts failed
+      attempts by default (they still consume budget); pass
+      `include_failed=False` there to count only successful calls.
+    - A deferred call (`ApprovalRequired` / `CallDeferred`) is **never**
+      counted: it did not execute in this run.
+    - All matching spans in the captured trace are counted, including tool
+      calls made by nested sub-agents (agent-as-tool delegation). If you
+      delegate to sub-agents that call their own tools, account for those
+      calls in your expectations and budgets.
+
+## Overview
+
+Agentic evaluators answer a class of "did the agent do the right thing?" questions that pure input/output checks can't:
+
+- **Tool coverage** — did the agent call the specific tools it was supposed to? ([`ToolCorrectness`][pydantic_evals.evaluators.ToolCorrectness])
+- **Trajectory shape** — did it call them in the right order, or at least use the right set? ([`TrajectoryMatch`][pydantic_evals.evaluators.TrajectoryMatch])
+- **Argument quality** — did the tool receive the expected inputs? ([`ArgumentCorrectness`][pydantic_evals.evaluators.ArgumentCorrectness])
+- **Budget discipline** — did the agent finish within a tool-call and/or model-request budget? ([`MaxToolCalls`][pydantic_evals.evaluators.MaxToolCalls], [`MaxModelRequests`][pydantic_evals.evaluators.MaxModelRequests])
+
+They are all deterministic, never call an LLM, and are cheap enough to run on every case in every experiment.
+
+## ToolCorrectness
+
+Assert that the agent called a specific **multiset** of tools. Repeated names require repeated calls.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import ToolCorrectness
+
+dataset = Dataset(
+    name='rag_agent',
+    cases=[Case(inputs='Summarize the latest papers on X')],
+    evaluators=[
+        ToolCorrectness(
+            expected_tools=['search', 'rerank', 'generate'],
+        ),
+    ],
+)
+```
+
+**Parameters:**
+
+- `expected_tools` (`list[str]`): Tool names the agent is expected to call. Order doesn't matter; duplicates are significant — `['search', 'search']` requires two `search` calls.
+- `allow_extra` (`bool`, default `False`): By default, any tool call not listed in `expected_tools` fails the check. Set to `True` to only require that the expected tools were called, permitting extras.
+- `include_failed` (`bool`, default `False`): Whether to count tool-call attempts that ended in an error.
+- `evaluation_name` (`str | None`): Custom name in reports.
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] with a `bool` value. The `reason` names missing and unexpected tools.
+
+## TrajectoryMatch
+
+Compare the actual ordered list of tool names to an expected one, using one of three modes.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import TrajectoryMatch
+
+dataset = Dataset(
+    name='ordered_tools',
+    cases=[Case(inputs='Process and file this request')],
+    evaluators=[
+        TrajectoryMatch(
+            expected_trajectory=['validate', 'enrich', 'submit'],
+            order='in_order',
+        ),
+    ],
+)
+```
+
+**Parameters:**
+
+- `expected_trajectory` (`list[str]`): Expected ordered list of tool names.
+- `order` (`Literal['exact', 'in_order', 'any_order']`, default `'in_order'`):
+    - `'exact'` — `1.0` iff the sequences are equal, else `0.0`.
+    - `'in_order'` — F1 computed from the longest common subsequence (LCS). Precision = `LCS / len(actual)`, recall = `LCS / len(expected)`. Allows extra calls interleaved with the expected order, but they reduce precision.
+    - `'any_order'` — F1 computed from the multiset intersection. Precision = `overlap / len(actual)`, recall = `overlap / len(expected)`. Order is ignored, but extra and missing calls both reduce the score.
+- `include_failed` (`bool`, default `False`): Whether the trajectory includes tool-call attempts that ended in an error.
+- `evaluation_name` (`str | None`): Custom name in reports.
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] with a `float` value in `[0.0, 1.0]`. For the F1-based modes, the reason text spells out the overlap, precision, recall, and F1 so the score is reproducible from the mismatch.
+
+For example, if `expected = ['a', 'b', 'c']` and the agent called `['a', 'x', 'b']`, the LCS is `['a', 'b']` (length 2), giving precision `2/3`, recall `2/3`, and F1 `≈ 0.667`.
+
+If both the expected and actual trajectories are empty, all modes score `1.0`; if only one of them is empty, all modes score `0.0`.
+
+## ArgumentCorrectness
+
+Check that a specific tool call received particular arguments.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import ArgumentCorrectness
+
+dataset = Dataset(
+    name='support_agent',
+    cases=[Case(inputs='Refund order 12345')],
+    evaluators=[
+        ArgumentCorrectness(
+            tool_name='issue_refund',
+            expected_arguments={'order_id': '12345'},
+            match_mode='subset',
+            occurrence='first',
+        ),
+    ],
+)
+```
+
+**Parameters:**
+
+- `tool_name` (`str`): The tool to inspect.
+- `expected_arguments` (`dict[str, Any]`): Expected argument keys/values.
+- `match_mode` (`Literal['exact', 'subset']`, default `'subset'`):
+    - `'subset'` — every expected key/value is present in the actual arguments. Note that this applies only to top-level keys: an expected *value* (including a nested dict) must compare equal to the actual value in full.
+    - `'exact'` — deep equality; unexpected keys also fail.
+- `occurrence` (`Literal['first', 'last'] | int`, default `'first'`): Which invocation to inspect if the tool is called multiple times. Integer indexes are 0-based.
+- `include_failed` (`bool`, default `False`): Whether tool-call attempts that ended in an error are considered. When `True`, each attempt counts as a separate occurrence.
+- `evaluation_name` (`str | None`): Custom name in reports.
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] with a `bool` value.
+
+**Graceful degradation:** this evaluator doesn't crash when arguments aren't available — for example, when the agent was instrumented with `include_content=False`, the evaluator returns `False` with a reason explaining the situation so your reports still make sense.
+
+## MaxToolCalls and MaxModelRequests
+
+Assert that the agent stayed within a tool-call and/or model-request budget. These follow the same shape as [`MaxDuration`][pydantic_evals.evaluators.MaxDuration]: one budget per evaluator, each reported as its own boolean assertion.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import MaxModelRequests, MaxToolCalls
+
+dataset = Dataset(
+    name='budget_aware',
+    cases=[Case(inputs='Draft a short reply')],
+    evaluators=[
+        MaxToolCalls(max_calls=5),
+        MaxModelRequests(max_requests=3),
+    ],
+)
+```
+
+**Parameters:**
+
+- `MaxToolCalls`: `max_calls` (`int`) — maximum allowed locally-executed tool calls. `include_failed` (`bool`, default `True`) controls whether attempts that ended in an error count against the budget (by default they do — they still consumed time and tokens).
+- `MaxModelRequests`: `max_requests` (`int`) — maximum allowed model (chat) requests. Prefers the `requests` value from `ctx.metrics` when available, otherwise counts LLM request spans directly (both use the same criteria).
+- Both accept `evaluation_name` (`str | None`) to customize the name in reports — useful when the same budget check appears at both the dataset and case level.
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] with a `bool` value. The `reason` includes the observed count and the budget.
+
+## Recipes
+
+### RAG agent
+
+Check that the retrieval pipeline runs *search → rerank → generate*, with no unexpected tool calls.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import ToolCorrectness, TrajectoryMatch
+
+dataset = Dataset(
+    name='rag_pipeline',
+    cases=[Case(inputs='Find papers on in-context learning')],
+    evaluators=[
+        ToolCorrectness(
+            expected_tools=['search', 'rerank', 'generate'],
+        ),
+        TrajectoryMatch(
+            expected_trajectory=['search', 'rerank', 'generate'],
+            order='exact',
+        ),
+    ],
+)
+```
+
+### Multi-tool agent where order matters
+
+Allow occasional retries, but require the main steps to happen in order.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import TrajectoryMatch
+
+dataset = Dataset(
+    name='ordered_with_slack',
+    cases=[Case(inputs='Process shipment 99')],
+    evaluators=[
+        TrajectoryMatch(
+            expected_trajectory=['validate', 'enrich', 'submit'],
+            order='in_order',  # F1-based: extra calls only reduce precision, order must be preserved
+        ),
+    ],
+)
+```
+
+### Support agent with `ArgumentCorrectness` and budget checks
+
+Verify that the right action was taken with the right inputs — within a reasonable number of steps.
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import (
+    ArgumentCorrectness,
+    MaxModelRequests,
+    MaxToolCalls,
+)
+
+dataset = Dataset(
+    name='refund_handling',
+    cases=[
+        Case(
+            name='valid_refund',
+            inputs={'query': 'Refund my order', 'order_id': '12345'},
+            evaluators=[
+                ArgumentCorrectness(
+                    tool_name='issue_refund',
+                    expected_arguments={'order_id': '12345'},
+                ),
+            ],
+        ),
+    ],
+    evaluators=[
+        MaxToolCalls(max_calls=4),
+        MaxModelRequests(max_requests=2),
+    ],
+)
+```
+
+### Task completion judged with the tool-call trajectory
+
+For tasks where deterministic checks aren't enough, you can have an LLM judge
+the task outcome together with the tool-call trajectory.
+[`LLMJudge`][pydantic_evals.evaluators.LLMJudge] only sees the case inputs,
+output, and expected output — not other evaluators' results or the span tree —
+so to give the judge visibility into *how* the agent got there, write a small
+custom evaluator that extracts the trajectory from the span tree and passes it
+to [`judge_input_output`][pydantic_evals.evaluators.llm_as_a_judge.judge_input_output]
+directly:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output
+from pydantic_evals.otel import SpanTreeRecordingError
+
+
+@dataclass
+class TrajectoryJudge(Evaluator):
+    rubric: str
+
+    async def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        try:
+            span_tree = ctx.span_tree
+        except SpanTreeRecordingError:
+            # Degrade gracefully, like the built-in evaluators on this page.
+            return EvaluationReason(value=False, reason='No span tree available.')
+
+        # Build a plain-text trajectory summary, mirroring what the built-in
+        # evaluators count as a tool call by default: tool spans are named
+        # 'running tool' (v2) or 'execute_tool {name}' (v3+); deferred calls
+        # never ran; output functions share the tool span shape but aren't
+        # tool calls; and failed attempts (status 'error') are dropped, like
+        # the built-in evaluators' `include_failed=False` default.
+        tool_names = [
+            node.attributes['gen_ai.tool.name']
+            for node in span_tree
+            if 'gen_ai.tool.name' in node.attributes
+            and 'pydantic_ai.tool.deferral.name' not in node.attributes
+            and node.status != 'error'
+            and (node.name == 'running tool' or node.name.startswith('execute_tool '))
+            and not str(node.attributes.get('logfire.msg', '')).startswith('running output function:')
+        ]
+        trajectory = ', '.join(str(n) for n in tool_names) or '(none)'
+        grading_output = await judge_input_output(
+            {'query': ctx.inputs, 'tool_trajectory': trajectory},
+            ctx.output,
+            self.rubric,
+        )
+        return EvaluationReason(value=grading_output.pass_, reason=grading_output.reason)
+
+
+dataset = Dataset(
+    name='task_completion',
+    cases=[Case(inputs='Resolve ticket 42')],
+    evaluators=[
+        TrajectoryJudge(
+            rubric=(
+                'The agent completed the task correctly, and the tool trajectory '
+                'included in the input is reasonable for the given query.'
+            ),
+        ),
+    ],
+)
+```
+
+This pattern keeps the deterministic checks above cheap and reproducible, and
+reserves the qualitative, open-ended judgement for the LLM — with the
+trajectory explicitly included in what the judge sees.
+
+## Next steps
+
+- [Span-Based Evaluation](span-based.md) — low-level span queries via `HasMatchingSpan` and `SpanQuery`
+- [Custom Evaluators](custom.md) — write your own evaluation logic
+- [Built-in Evaluators](built-in.md) — complete reference of other evaluator types

docs/evals/evaluators/built-in.md

@@ -0,0 +1,524 @@
+# Native Evaluators
+
+Pydantic Evals provides several built-in evaluators for common evaluation tasks.
+
+## Comparison Evaluators
+
+### EqualsExpected
+
+Check if the output exactly equals the expected output from the case.
+
+```python
+from pydantic_evals.evaluators import EqualsExpected
+
+EqualsExpected()
+```
+
+**Parameters:** None
+
+**Returns:** `bool` - `True` if `ctx.output == ctx.expected_output`
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import EqualsExpected
+
+dataset = Dataset(
+    name='equals_expected_demo',
+    cases=[
+        Case(
+            name='addition',
+            inputs='2 + 2',
+            expected_output='4',
+        ),
+    ],
+    evaluators=[EqualsExpected()],
+)
+```
+
+**Notes:**
+
+- Skips evaluation if `expected_output` is `None` (returns empty dict `{}`)
+- Uses Python's `==` operator, so works with any comparable types
+- For structured data, considers nested equality
+
+---
+
+### Equals
+
+Check if the output equals a specific value.
+
+```python
+from pydantic_evals.evaluators import Equals
+
+Equals(value='expected_result')
+```
+
+**Parameters:**
+
+- `value` (Any): The value to compare against
+- `evaluation_name` (str | None): Custom name for this evaluation in reports
+
+**Returns:** `bool` - `True` if `ctx.output == value`
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import Equals
+
+# Check output is always "success"
+dataset = Dataset(
+    name='equals_demo',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        Equals(value='success', evaluation_name='is_success'),
+    ],
+)
+```
+
+**Use Cases:**
+
+- Checking for sentinel values
+- Validating consistent outputs
+- Testing classification into specific categories
+
+---
+
+### Contains
+
+Check if the output contains a specific value or substring.
+
+```python
+from pydantic_evals.evaluators import Contains
+
+Contains(
+    value='substring',
+    case_sensitive=True,
+    as_strings=False,
+)
+```
+
+**Parameters:**
+
+- `value` (Any): The value to search for
+- `case_sensitive` (bool): Case-sensitive comparison for strings (default: `True`)
+- `as_strings` (bool): Convert both values to strings before checking (default: `False`)
+- `evaluation_name` (str | None): Custom name for this evaluation in reports
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] - Pass/fail with explanation
+
+**Behavior:**
+
+For **strings**: checks substring containment
+
+- `Contains(value='hello', case_sensitive=False)`
+  - Matches: "Hello World", "say hello", "HELLO"
+  - Doesn't match: "hi there"
+
+For **lists/tuples**: checks membership
+
+- `Contains(value='apple')`
+  - Matches: `['apple', 'banana']`, `('apple',)`
+  - Doesn't match: `['apples', 'orange']`
+
+For **dicts**: checks key-value pairs
+
+- `Contains(value={'name': 'Alice'})`
+  - Matches: `{'name': 'Alice', 'age': 30}`
+  - Doesn't match: `{'name': 'Bob'}`
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import Contains
+
+dataset = Dataset(
+    name='contains_demo',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # Check for required keywords
+        Contains(value='terms and conditions', case_sensitive=False),
+        # Check for PII (fail if found)
+        # Note: Use a custom evaluator that returns False when PII found
+    ],
+)
+```
+
+**Use Cases:**
+
+- Required content verification
+- Keyword detection
+- PII/sensitive data detection
+- Multi-value validation
+
+---
+
+## Type Validation
+
+### IsInstance
+
+Check if the output is an instance of a type with the given name.
+
+```python
+from pydantic_evals.evaluators import IsInstance
+
+IsInstance(type_name='str')
+```
+
+**Parameters:**
+
+- `type_name` (str): The type name to check (uses `__name__` or `__qualname__`)
+- `evaluation_name` (str | None): Custom name for this evaluation in reports
+
+**Returns:** [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] - Pass/fail with type information
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import IsInstance
+
+dataset = Dataset(
+    name='isinstance_demo',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # Check output is always a string
+        IsInstance(type_name='str'),
+        # Check for Pydantic model
+        IsInstance(type_name='MyModel'),
+        # Check for dict
+        IsInstance(type_name='dict'),
+    ],
+)
+```
+
+**Notes:**
+
+- Matches against both `__name__` and `__qualname__` of the type
+- Works with built-in types (`str`, `int`, `dict`, `list`, etc.)
+- Works with custom classes and Pydantic models
+- Checks the entire MRO (Method Resolution Order) for inheritance
+
+**Use Cases:**
+
+- Format validation
+- Structured output verification
+- Type consistency checks
+
+---
+
+## Performance Evaluation
+
+### MaxDuration
+
+Check if task execution time is under a maximum threshold.
+
+```python
+from datetime import timedelta
+
+from pydantic_evals.evaluators import MaxDuration
+
+MaxDuration(seconds=2.0)
+# or
+MaxDuration(seconds=timedelta(seconds=2))
+```
+
+**Parameters:**
+
+- `seconds` (float | timedelta): Maximum allowed duration
+
+**Returns:** `bool` - `True` if `ctx.duration <= seconds`
+
+**Example:**
+
+```python
+from datetime import timedelta
+
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import MaxDuration
+
+dataset = Dataset(
+    name='max_duration_demo',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # SLA: must respond in under 2 seconds
+        MaxDuration(seconds=2.0),
+        # Or using timedelta
+        MaxDuration(seconds=timedelta(milliseconds=500)),
+    ],
+)
+```
+
+**Use Cases:**
+
+- SLA compliance
+- Performance regression testing
+- Latency requirements
+- Timeout validation
+
+**See Also:** [Concurrency & Performance](../how-to/concurrency.md)
+
+---
+
+## LLM-as-a-Judge
+
+### LLMJudge
+
+Use an LLM to evaluate subjective qualities based on a rubric.
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='Response is accurate and helpful',
+    model='openai:gpt-5.2',
+    include_input=False,
+    include_expected_output=False,
+    model_settings=None,
+    score=False,
+    assertion={'include_reason': True},
+)
+```
+
+**Parameters:**
+
+- `rubric` (str): The evaluation criteria (required)
+- `model` (Model | KnownModelName | None): Model to use (default: `'openai:gpt-5.2'`)
+- `include_input` (bool): Include task inputs in the prompt (default: `False`)
+- `include_expected_output` (bool): Include expected output in the prompt (default: `False`)
+- `model_settings` (ModelSettings | None): Custom model settings
+- `score` (OutputConfig | False): Configure score output (default: `False`)
+- `assertion` (OutputConfig | False): Configure assertion output (default: includes reason)
+
+**Returns:** Depends on `score` and `assertion` parameters (see below)
+
+**Output Modes:**
+
+By default, returns a **boolean assertion** with reason:
+
+- `LLMJudge(rubric='Response is polite')`
+  - Returns: `{'LLMJudge_pass': EvaluationReason(value=True, reason='...')}`
+
+Return a **score** (0.0 to 1.0) instead:
+
+- `LLMJudge(rubric='Response quality', score={'include_reason': True}, assertion=False)`
+  - Returns: `{'LLMJudge_score': EvaluationReason(value=0.85, reason='...')}`
+
+Return **both** score and assertion:
+
+- `LLMJudge(rubric='Response quality', score={'include_reason': True}, assertion={'include_reason': True})`
+  - Returns: `{'LLMJudge_score': EvaluationReason(value=0.85, reason='...'), 'LLMJudge_pass': EvaluationReason(value=True, reason='...')}`
+
+**Customize evaluation names:**
+
+- `LLMJudge(rubric='Response is factually accurate', assertion={'evaluation_name': 'accuracy', 'include_reason': True})`
+  - Returns: `{'accuracy': EvaluationReason(value=True, reason='...')}`
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='llm_judge_demo',
+    cases=[Case(inputs='test', expected_output='result')],
+    evaluators=[
+        # Basic accuracy check
+        LLMJudge(
+            rubric='Response is factually accurate',
+            include_input=True,
+        ),
+        # Quality score with different model
+        LLMJudge(
+            rubric='Overall response quality',
+            model='anthropic:claude-sonnet-4-6',
+            score={'evaluation_name': 'quality', 'include_reason': False},
+            assertion=False,
+        ),
+        # Check against expected output
+        LLMJudge(
+            rubric='Response matches the expected answer semantically',
+            include_input=True,
+            include_expected_output=True,
+        ),
+    ],
+)
+```
+
+**See Also:** [LLM Judge Deep Dive](llm-judge.md)
+
+### GEval
+
+Chain-of-thought evaluation following the G-Eval method (Liu et al., 2023): the judge applies
+explicit evaluation steps and returns an integer score in `score_range` with a reasoning trace.
+
+```python
+from pydantic_evals.evaluators import GEval
+
+GEval(
+    criteria='coherence',
+    evaluation_steps=[
+        'Read the output carefully.',
+        'Check that each sentence follows logically from the previous one.',
+        'Assign a score from 1 (incoherent) to 5 (fully coherent).',
+    ],
+    score_range=(1, 5),
+    include_input=False,
+)
+```
+
+**Parameters:**
+
+- `criteria` (str): The aspect being evaluated, e.g. `'coherence'` (required)
+- `evaluation_steps` (list[str]): Explicit chain-of-thought steps the judge should follow (required)
+- `score_range` (tuple[int, int]): Inclusive integer score range (default: `(1, 5)`)
+- `include_input` (bool): Include task inputs in the prompt (default: `False`)
+- `model` (Model | KnownModelName | None): Model to use (default: `'openai:gpt-5.2'`)
+- `model_settings` (ModelSettings | None): Custom model settings
+- `evaluation_name` (str | None): Custom name for the result (default: `'GEval'`)
+
+**Returns:** `EvaluationReason` with the integer score and the judge's reasoning
+
+**See Also:** [Standard Quality Metrics](standard-quality-metrics.md)
+
+---
+
+## Span-Based Evaluation
+
+### HasMatchingSpan
+
+Check if OpenTelemetry spans match a query (requires Logfire configuration).
+
+```python
+from pydantic_evals.evaluators import HasMatchingSpan
+
+HasMatchingSpan(
+    query={'name_contains': 'tool_call'},
+    evaluation_name='called_tool',
+)
+```
+
+**Parameters:**
+
+- `query` ([`SpanQuery`][pydantic_evals.otel.SpanQuery]): Query to match against spans
+- `evaluation_name` (str | None): Custom name for this evaluation in reports
+
+**Returns:** `bool` - `True` if any span matches the query
+
+**Example:**
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import HasMatchingSpan
+
+dataset = Dataset(
+    name='span_check_demo',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # Check that a specific tool was called
+        HasMatchingSpan(
+            query={'name_contains': 'search_database'},
+            evaluation_name='used_database',
+        ),
+        # Check for errors
+        HasMatchingSpan(
+            query={'has_status': 'error'},
+            evaluation_name='had_errors',
+        ),
+        # Check duration constraints
+        HasMatchingSpan(
+            query={
+                'name_equals': 'llm_call',
+                'max_duration': 2.0,  # seconds
+            },
+            evaluation_name='llm_fast_enough',
+        ),
+    ],
+)
+```
+
+**See Also:** [Span-Based Evaluation](span-based.md)
+
+---
+
+## Native Report Evaluators
+
+In addition to the case-level evaluators above, Pydantic Evals provides report evaluators that
+analyze entire experiment results. These are passed via the `report_evaluators` parameter on `Dataset`.
+
+| Report Evaluator | Purpose | Output |
+|------------------|---------|--------|
+| [`ConfusionMatrixEvaluator`][pydantic_evals.evaluators.ConfusionMatrixEvaluator] | Classification confusion matrix | `ConfusionMatrix` |
+| [`PrecisionRecallEvaluator`][pydantic_evals.evaluators.PrecisionRecallEvaluator] | PR curve with AUC | `PrecisionRecall` |
+
+**See:** [Report Evaluators](report-evaluators.md) for full documentation, parameters, and examples,
+including how to write custom report evaluators that produce `ScalarResult` and `TableResult` analyses.
+
+---
+
+## Quick Reference Table
+
+### Case-Level Evaluators
+
+| Evaluator | Purpose | Return Type | Cost | Speed |
+|-----------|---------|-------------|------|-------|
+| [`EqualsExpected`][pydantic_evals.evaluators.EqualsExpected] | Exact match with expected | `bool` | Free | Instant |
+| [`Equals`][pydantic_evals.evaluators.Equals] | Equals specific value | `bool` | Free | Instant |
+| [`Contains`][pydantic_evals.evaluators.Contains] | Contains value/substring | `bool` + reason | Free | Instant |
+| [`IsInstance`][pydantic_evals.evaluators.IsInstance] | Type validation | `bool` + reason | Free | Instant |
+| [`MaxDuration`][pydantic_evals.evaluators.MaxDuration] | Performance threshold | `bool` | Free | Instant |
+| [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] | Subjective quality | `bool` and/or `float` | $$ | Slow |
+| [`GEval`][pydantic_evals.evaluators.GEval] | Chain-of-thought scoring | `int` + reason | $$ | Slow |
+| [`HasMatchingSpan`][pydantic_evals.evaluators.HasMatchingSpan] | Behavioral check | `bool` | Free | Fast |
+
+### Report-Level Evaluators
+
+| Evaluator | Purpose | Output Type | Cost | Speed |
+|-----------|---------|-------------|------|-------|
+| [`ConfusionMatrixEvaluator`][pydantic_evals.evaluators.ConfusionMatrixEvaluator] | Classification matrix | `ConfusionMatrix` | Free | Instant |
+| [`PrecisionRecallEvaluator`][pydantic_evals.evaluators.PrecisionRecallEvaluator] | PR curve with AUC | `PrecisionRecall` | Free | Instant |
+
+## Combining Evaluators
+
+Best practice is to combine fast deterministic checks with slower LLM evaluations:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import (
+    Contains,
+    IsInstance,
+    LLMJudge,
+    MaxDuration,
+)
+
+dataset = Dataset(
+    name='combined_evaluators',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # Fast checks first (fail fast)
+        IsInstance(type_name='str'),
+        Contains(value='required_field'),
+        MaxDuration(seconds=2.0),
+        # Expensive LLM checks last
+        LLMJudge(rubric='Response is helpful and accurate'),
+    ],
+)
+```
+
+This approach:
+
+1. Catches format/structure issues immediately
+2. Validates required content quickly
+3. Only runs expensive LLM evaluation if basic checks pass
+4. Provides comprehensive quality assessment
+
+## Next Steps
+
+- **[LLM Judge](llm-judge.md)** - Deep dive on LLM-as-a-Judge evaluation
+- **[Custom Evaluators](custom.md)** - Write your own evaluation logic
+- **[Report Evaluators](report-evaluators.md)** - Experiment-wide analyses (confusion matrices, PR curves, etc.)
+- **[Span-Based Evaluation](span-based.md)** - Using OpenTelemetry spans for behavioral checks

docs/evals/evaluators/custom.md

@@ -0,0 +1,811 @@
+# Custom Evaluators
+
+Write custom evaluators for domain-specific logic, external integrations, or specialized metrics.
+
+## Basic Custom Evaluator
+
+All evaluators inherit from [`Evaluator`][pydantic_evals.evaluators.Evaluator] and must implement `evaluate`:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ExactMatch(Evaluator):
+    """Check if output exactly matches expected output."""
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return ctx.output == ctx.expected_output
+```
+
+**Key Points:**
+
+- Use `@dataclass` decorator (required)
+- Inherit from `Evaluator`
+- Implement `evaluate(self, ctx: EvaluatorContext) -> EvaluatorOutput`
+- Return `bool`, `int`, `float`, `str`, [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason], or `dict` of these
+
+## EvaluatorContext
+
+The context provides all information about the case execution:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class MyEvaluator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        # Access case data
+        ctx.name              # Case name
+        ctx.inputs            # Task inputs
+        ctx.metadata          # Case metadata
+        ctx.expected_output   # Expected output (may be None)
+        ctx.output            # Actual output
+
+        # Performance data
+        ctx.duration          # Task execution time (seconds)
+
+        # Custom metrics/attributes (see metrics guide)
+        ctx.metrics           # dict[str, int | float]
+        ctx.attributes        # dict[str, Any]
+
+        # OpenTelemetry spans (if logfire configured)
+        ctx.span_tree         # SpanTree for behavioral checks
+
+        return True
+```
+
+## Evaluator Parameters
+
+Add configurable parameters as dataclass fields:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ContainsKeyword(Evaluator):
+    keyword: str
+    case_sensitive: bool = True
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        output = ctx.output
+        keyword = self.keyword
+
+        if not self.case_sensitive:
+            output = output.lower()
+            keyword = keyword.lower()
+
+        return keyword in output
+
+
+# Usage
+dataset = Dataset(
+    name='keyword_check',
+    cases=[Case(name='test', inputs='This is important')],
+    evaluators=[
+        ContainsKeyword(keyword='important', case_sensitive=False),
+    ],
+)
+```
+
+## Return Types
+
+### Boolean Assertions
+
+Simple pass/fail checks:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class IsValidJSON(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        try:
+            import json
+            json.loads(ctx.output)
+            return True
+        except Exception:
+            return False
+```
+
+### Numeric Scores
+
+Quality metrics:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class LengthScore(Evaluator):
+    """Score based on output length (0.0 = too short, 1.0 = ideal)."""
+
+    ideal_length: int = 100
+    tolerance: int = 20
+
+    def evaluate(self, ctx: EvaluatorContext) -> float:
+        length = len(ctx.output)
+        diff = abs(length - self.ideal_length)
+
+        if diff <= self.tolerance:
+            return 1.0
+        else:
+            # Decay score as we move away from ideal
+            score = max(0.0, 1.0 - (diff - self.tolerance) / self.ideal_length)
+            return score
+```
+
+!!! note "Scores must be finite"
+    Numeric scores have to be finite. An evaluator that returns `NaN` or `±inf` — as a scalar, inside an [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason], or as a value in a returned dict — produces an [`EvaluatorFailure`][pydantic_evals.evaluators.EvaluatorFailure] for that case instead of a score, so a non-comparable value is surfaced as a failed evaluator rather than silently recorded.
+
+### String Labels
+
+Categorical classifications:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class SentimentClassifier(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> str:
+        output_lower = ctx.output.lower()
+
+        if any(word in output_lower for word in ['error', 'failed', 'wrong']):
+            return 'negative'
+        elif any(word in output_lower for word in ['success', 'correct', 'great']):
+            return 'positive'
+        else:
+            return 'neutral'
+```
+
+### With Reasons
+
+Add explanations to any result:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class SmartCheck(Evaluator):
+    threshold: float = 0.8
+
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        score = self._calculate_score(ctx.output)
+
+        if score >= self.threshold:
+            return EvaluationReason(
+                value=True,
+                reason=f'Score {score:.2f} exceeds threshold {self.threshold}',
+            )
+        else:
+            return EvaluationReason(
+                value=False,
+                reason=f'Score {score:.2f} below threshold {self.threshold}',
+            )
+
+    def _calculate_score(self, output: str) -> float:
+        # Your scoring logic
+        return 0.75
+```
+
+### Multiple Results
+
+You can return multiple evaluations from one evaluator by returning a dictionary of key-value pairs.
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import (
+    EvaluationReason,
+    Evaluator,
+    EvaluatorContext,
+    EvaluatorOutput,
+)
+
+
+@dataclass
+class ComprehensiveCheck(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluatorOutput:
+        format_valid = self._check_format(ctx.output)
+
+        return {
+            'valid_format': EvaluationReason(
+                value=format_valid,
+                reason='Valid JSON format' if format_valid else 'Invalid JSON format',
+            ),
+            'quality_score': self._score_quality(ctx.output),  # float
+            'category': self._classify(ctx.output),  # str
+        }
+
+    def _check_format(self, output: str) -> bool:
+        return output.startswith('{') and output.endswith('}')
+
+    def _score_quality(self, output: str) -> float:
+        return len(output) / 100.0
+
+    def _classify(self, output: str) -> str:
+        return 'short' if len(output) < 50 else 'long'
+```
+
+Each key in the returned dictionary becomes a separate result in the report. Values can be:
+
+- Primitives (`bool`, `int`, `float`, `str`)
+- [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason] (value with explanation)
+- Nested dicts of these types
+
+The [`EvaluatorOutput`][pydantic_evals.evaluators.evaluator.EvaluatorOutput] type represents all legal values
+that can be returned by an evaluator, and can be used as the return type annotation for your custom `evaluate` method.
+
+### Conditional Results
+
+Evaluators can dynamically choose whether to produce results for a given case by returning an empty dict when not applicable:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import (
+    EvaluationReason,
+    Evaluator,
+    EvaluatorContext,
+    EvaluatorOutput,
+)
+
+
+@dataclass
+class SQLValidator(Evaluator):
+    """Only evaluates SQL queries, skips other outputs."""
+
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluatorOutput:
+        # Check if this case is relevant for SQL validation
+        if not isinstance(ctx.output, str) or not ctx.output.strip().upper().startswith(
+            ('SELECT', 'INSERT', 'UPDATE', 'DELETE')
+        ):
+            # Return empty dict - this evaluator doesn't apply to this case
+            return {}
+
+        # This is a SQL query, perform validation
+        try:
+            # In real implementation, use sqlparse or similar
+            is_valid = self._validate_sql(ctx.output)
+            return {
+                'sql_valid': is_valid,
+                'sql_complexity': self._measure_complexity(ctx.output),
+            }
+        except Exception as e:
+            return {'sql_valid': EvaluationReason(False, reason=f'Exception: {e}')}
+
+    def _validate_sql(self, query: str) -> bool:
+        # Simplified validation
+        return 'FROM' in query.upper() or 'INTO' in query.upper()
+
+    def _measure_complexity(self, query: str) -> str:
+        joins = query.upper().count('JOIN')
+        if joins == 0:
+            return 'simple'
+        elif joins <= 2:
+            return 'moderate'
+        else:
+            return 'complex'
+```
+
+This pattern is useful when:
+
+- An evaluator only applies to certain types of outputs (e.g., code validation only for code outputs)
+- Validation depends on metadata tags (e.g., only evaluate cases marked with `language='python'`)
+- You want to run expensive checks conditionally based on other evaluator results
+
+**Key Points:**
+
+- Returning `{}` means "this evaluator doesn't apply here" - the case won't show results from this evaluator
+- Returning `{'key': value}` means "this evaluator applies and here are the results"
+- This is more practical than using case-level evaluators when it applies to a large fraction of cases, or when the
+  condition is based on the output itself
+- The evaluator still runs for every case, but can short-circuit when not relevant
+
+## Async Evaluators
+
+Use `async def` for I/O-bound operations:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class APIValidator(Evaluator):
+    api_url: str
+
+    async def evaluate(self, ctx: EvaluatorContext) -> bool:
+        import httpx
+
+        async with httpx.AsyncClient() as client:
+            response = await client.post(
+                self.api_url,
+                json={'output': ctx.output},
+            )
+            return response.json()['valid']
+```
+
+Pydantic Evals handles both sync and async evaluators automatically.
+
+## Using Metadata
+
+Access case metadata for context-aware evaluation:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class DifficultyAwareScore(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> float:
+        # Base score
+        base_score = self._score_output(ctx.output)
+
+        # Adjust based on difficulty from metadata
+        if ctx.metadata and 'difficulty' in ctx.metadata:
+            difficulty = ctx.metadata['difficulty']
+
+            if difficulty == 'easy':
+                # Penalize mistakes more on easy questions
+                return base_score
+            elif difficulty == 'hard':
+                # Be more lenient on hard questions
+                return min(1.0, base_score * 1.2)
+
+        return base_score
+
+    def _score_output(self, output: str) -> float:
+        # Your scoring logic
+        return 0.8
+```
+
+## Using Metrics
+
+Access custom metrics set during task execution:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals import increment_eval_metric, set_eval_attribute
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+# In your task
+def my_task(inputs: str) -> str:
+    result = f'processed: {inputs}'
+
+    # Record metrics
+    increment_eval_metric('api_calls', 3)
+    set_eval_attribute('used_cache', True)
+
+    return result
+
+
+# In your evaluator
+@dataclass
+class EfficiencyCheck(Evaluator):
+    max_api_calls: int = 5
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        api_calls = ctx.metrics.get('api_calls', 0)
+        return api_calls <= self.max_api_calls
+```
+
+See [Metrics & Attributes Guide](../how-to/metrics-attributes.md) for more.
+
+## Generic Type Parameters
+
+Make evaluators type-safe with generics:
+
+```python
+from dataclasses import dataclass
+from typing import TypeVar
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+InputsT = TypeVar('InputsT')
+OutputT = TypeVar('OutputT')
+
+
+@dataclass
+class TypedEvaluator(Evaluator[InputsT, OutputT, dict]):
+    def evaluate(self, ctx: EvaluatorContext[InputsT, OutputT, dict]) -> bool:
+        # ctx.inputs and ctx.output are now properly typed
+        return True
+```
+
+## Custom Evaluation Names
+
+Control how evaluations appear in reports:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class CustomNameEvaluator(Evaluator):
+    check_type: str
+
+    def get_default_evaluation_name(self) -> str:
+        # Use check_type as the name instead of class name
+        return f'{self.check_type}_check'
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return True
+
+
+# In reports, appears as "format_check" instead of "CustomNameEvaluator"
+evaluator = CustomNameEvaluator(check_type='format')
+```
+
+Or use the `evaluation_name` field (if using the built-in pattern):
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class MyEvaluator(Evaluator):
+    evaluation_name: str | None = None
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return True
+
+
+# Usage
+MyEvaluator(evaluation_name='my_custom_name')
+```
+
+## Real-World Examples
+
+### SQL Validation
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class ValidSQL(Evaluator):
+    dialect: str = 'postgresql'
+
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        try:
+            import sqlparse
+            parsed = sqlparse.parse(ctx.output)
+
+            if not parsed:
+                return EvaluationReason(
+                    value=False,
+                    reason='Could not parse SQL',
+                )
+
+            # Check for dangerous operations
+            sql_upper = ctx.output.upper()
+            if 'DROP' in sql_upper or 'DELETE' in sql_upper:
+                return EvaluationReason(
+                    value=False,
+                    reason='Contains dangerous operations (DROP/DELETE)',
+                )
+
+            return EvaluationReason(
+                value=True,
+                reason='Valid SQL syntax',
+            )
+        except Exception as e:
+            return EvaluationReason(
+                value=False,
+                reason=f'SQL parsing error: {e}',
+            )
+```
+
+### Code Execution
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class ExecutablePython(Evaluator):
+    timeout_seconds: float = 5.0
+
+    async def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        import asyncio
+        import os
+        import tempfile
+
+        # Write code to temp file
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+            f.write(ctx.output)
+            temp_path = f.name
+
+        try:
+            # Execute with timeout
+            process = await asyncio.create_subprocess_exec(
+                'python', temp_path,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+
+            try:
+                stdout, stderr = await asyncio.wait_for(
+                    process.communicate(),
+                    timeout=self.timeout_seconds,
+                )
+            except asyncio.TimeoutError:
+                process.kill()
+                return EvaluationReason(
+                    value=False,
+                    reason=f'Execution timeout after {self.timeout_seconds}s',
+                )
+
+            if process.returncode == 0:
+                return EvaluationReason(
+                    value=True,
+                    reason='Code executed successfully',
+                )
+            else:
+                return EvaluationReason(
+                    value=False,
+                    reason=f'Execution failed: {stderr.decode()}',
+                )
+        finally:
+            os.unlink(temp_path)
+```
+
+### External API Validation
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class APIResponseValid(Evaluator):
+    api_endpoint: str
+    api_key: str
+
+    async def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool | float]:
+        import httpx
+
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    self.api_endpoint,
+                    headers={'Authorization': f'Bearer {self.api_key}'},
+                    json={'data': ctx.output},
+                    timeout=10.0,
+                )
+
+                result = response.json()
+
+                return {
+                    'api_reachable': True,
+                    'validation_passed': result.get('valid', False),
+                    'confidence_score': result.get('confidence', 0.0),
+                }
+        except Exception:
+            return {
+                'api_reachable': False,
+                'validation_passed': False,
+                'confidence_score': 0.0,
+            }
+```
+
+## Testing Evaluators
+
+Test evaluators like any other Python code:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ExactMatch(Evaluator):
+    """Check if output exactly matches expected output."""
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return ctx.output == ctx.expected_output
+
+
+def test_exact_match():
+    evaluator = ExactMatch()
+
+    # Test match
+    ctx = EvaluatorContext(
+        name='test',
+        inputs='input',
+        metadata=None,
+        expected_output='expected',
+        output='expected',
+        duration=0.1,
+        _span_tree=None,
+        attributes={},
+        metrics={},
+    )
+    assert evaluator.evaluate(ctx) is True
+
+    # Test mismatch
+    ctx.output = 'different'
+    assert evaluator.evaluate(ctx) is False
+```
+
+## Best Practices
+
+### 1. Keep Evaluators Focused
+
+Each evaluator should check one thing:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+def check_format(output: str) -> bool:
+    return output.startswith('{')
+
+
+def check_content(output: str) -> bool:
+    return len(output) > 10
+
+
+def check_length(output: str) -> bool:
+    return len(output) < 1000
+
+
+def check_spelling(output: str) -> bool:
+    return True  # Placeholder
+
+
+# Bad: Doing too much
+@dataclass
+class EverythingChecker(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> dict:
+        return {
+            'format_valid': check_format(ctx.output),
+            'content_good': check_content(ctx.output),
+            'length_ok': check_length(ctx.output),
+            'spelling_correct': check_spelling(ctx.output),
+        }
+
+
+# Good: Separate evaluators
+@dataclass
+class FormatValidator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return check_format(ctx.output)
+
+
+@dataclass
+class ContentChecker(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return check_content(ctx.output)
+
+
+@dataclass
+class LengthChecker(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return check_length(ctx.output)
+
+
+@dataclass
+class SpellingChecker(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return check_spelling(ctx.output)
+```
+
+Some exceptions to this:
+
+* When there is a significant amount of shared computation or network request latency, it may be better to have a single evaluator calculate all dependent outputs together.
+* If multiple checks are tightly coupled or very closely related to each other, it may make sense to include all their logic in one evaluator.
+
+### 2. Handle Missing Data Gracefully
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class SafeEvaluator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        if ctx.expected_output is None:
+            return EvaluationReason(
+                value=True,
+                reason='Skipped: no expected output provided',
+            )
+
+        # Your evaluation logic
+        ...
+```
+
+### 3. Provide Helpful Reasons
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class HelpfulEvaluator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        # Bad
+        return EvaluationReason(value=False, reason='Failed')
+
+        # Good
+        return EvaluationReason(
+            value=False,
+            reason=f'Expected {ctx.expected_output!r}, got {ctx.output!r}',
+        )
+```
+
+### 4. Use Timeouts for External Calls
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class APIEvaluator(Evaluator):
+    timeout: float = 10.0
+
+    async def _call_api(self, output: str) -> bool:
+        # Placeholder for API call
+        return True
+
+    async def evaluate(self, ctx: EvaluatorContext) -> bool:
+        import asyncio
+
+        try:
+            return await asyncio.wait_for(
+                self._call_api(ctx.output),
+                timeout=self.timeout,
+            )
+        except asyncio.TimeoutError:
+            return False
+```
+
+## Next Steps
+
+- **[Report Evaluators](report-evaluators.md)** - Experiment-wide analyses (confusion matrices, PR curves, custom tables)
+- **[Span-Based Evaluation](span-based.md)** - Using OpenTelemetry spans
+- **[Examples](../examples/simple-validation.md)** - Practical examples

docs/evals/evaluators/framework-integrations.md

@@ -0,0 +1,145 @@
+# Third-Party Integrations
+
+Pydantic Evals does not take a hard dependency on any particular metrics framework. When a team
+already uses [Ragas](https://github.com/vibrantlabsai/ragas),
+[DeepEval](https://github.com/confident-ai/deepeval), or another scoring library, the
+[`Evaluator`][pydantic_evals.evaluators.Evaluator] base class makes it straightforward to wrap the
+upstream metric and run it inside any Pydantic Evals dataset. This page shows worked examples for
+the common ones.
+
+!!! tip "Prefer a native evaluator where you can"
+    If a rubric-based [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] (see the
+    [standard quality metrics](standard-quality-metrics.md) page for ready-made rubrics) or a
+    [custom evaluator](custom.md) covers your use case, that's usually simpler — zero extra
+    dependencies and the scores slot into reports cleanly. Reach for the integrations below
+    when you specifically want the *exact* upstream implementation (for reproducibility with
+    published benchmarks, parity with an existing evaluation suite, or features we don't
+    expose natively). You can mix external and native evaluators in one dataset.
+
+## Pattern
+
+Each framework integration follows the same pattern:
+
+1. Subclass [`Evaluator`][pydantic_evals.evaluators.Evaluator].
+2. Adapt `ctx.inputs`, `ctx.output`, `ctx.expected_output`, and metadata into whatever the
+   upstream metric expects.
+3. Return a `float` score, a `bool` assertion, an [`EvaluationReason`][pydantic_evals.evaluators.EvaluationReason],
+   or a `dict` of these.
+
+The rest of this page shows concrete adapters. They are intentionally compact — extend them with
+whatever configuration your team needs (model selection, thresholds, per-case toggles).
+
+## Ragas
+
+Install with `pip install ragas` (not included in `pydantic-evals`).
+
+This adapter wraps [`ragas.metrics.Faithfulness`](https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/faithfulness/)
+for a single-turn sample. Each case is expected to provide the retrieved context as part of its
+inputs or metadata.
+
+```python {test="skip" lint="skip"}
+from dataclasses import dataclass
+
+from ragas.dataset_schema import SingleTurnSample
+from ragas.metrics import Faithfulness
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class RagasFaithfulness(Evaluator):
+    """Wrap `ragas.metrics.Faithfulness` as a Pydantic Evals evaluator."""
+
+    context_field: str = 'context'
+
+    async def evaluate(self, ctx: EvaluatorContext) -> EvaluationReason:
+        metadata = ctx.metadata or {}
+        retrieved_contexts = metadata.get(self.context_field, [])
+        if isinstance(retrieved_contexts, str):
+            retrieved_contexts = [retrieved_contexts]
+
+        sample = SingleTurnSample(
+            user_input=str(ctx.inputs),
+            response=str(ctx.output),
+            retrieved_contexts=retrieved_contexts,
+        )
+        metric = Faithfulness()
+        score = await metric.single_turn_ascore(sample)
+        return EvaluationReason(value=float(score), reason=f'ragas.Faithfulness = {score:.3f}')
+```
+
+Usage is the same as any built-in evaluator:
+
+```python {test="skip" lint="skip"}
+from pydantic_evals import Case, Dataset
+
+dataset = Dataset(
+    name='rag_eval',
+    cases=[
+        Case(
+            inputs='What is the capital of France?',
+            metadata={'context': ['Paris is the capital of France.']},
+        ),
+    ],
+    evaluators=[RagasFaithfulness()],
+)
+```
+
+The same pattern works for `ragas.metrics.answer_relevancy`, `context_precision`, and the other
+scoring metrics: swap the metric class and (if needed) the sample fields.
+
+## DeepEval
+
+Install with `pip install deepeval` (not included in `pydantic-evals`).
+
+This adapter wraps [DeepEval's `GEval` metric](https://docs.confident-ai.com/docs/metrics-llm-evals)
+to score a criterion against a `LLMTestCase`. DeepEval's `measure` is synchronous, so the
+evaluator is synchronous too.
+
+```python {test="skip" lint="skip"}
+from dataclasses import dataclass
+
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+from pydantic_evals.evaluators import EvaluationReason, Evaluator, EvaluatorContext
+
+
+@dataclass
+class DeepEvalGEval(Evaluator):
+    """Wrap `deepeval.metrics.GEval` as a Pydantic Evals evaluator."""
+
+    metric_name: str
+    criteria: str
+    threshold: float = 0.5
+
+    def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool | EvaluationReason]:
+        test_case = LLMTestCase(
+            input=str(ctx.inputs),
+            actual_output=str(ctx.output),
+            expected_output=None if ctx.expected_output is None else str(ctx.expected_output),
+        )
+        metric = GEval(
+            name=self.metric_name,
+            criteria=self.criteria,
+            evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT],
+            threshold=self.threshold,
+        )
+        metric.measure(test_case)
+        return {
+            f'{self.metric_name}_score': EvaluationReason(value=float(metric.score), reason=metric.reason or ''),
+            f'{self.metric_name}_pass': bool(metric.success),
+        }
+```
+
+The same wrapper shape works for DeepEval's `FaithfulnessMetric`, `AnswerRelevancyMetric`,
+`HallucinationMetric`, and others — swap the metric class and populate the relevant
+`LLMTestCase` fields (for example `retrieval_context` for faithfulness).
+
+## Notes on dependencies
+
+- `ragas` and `deepeval` are optional dependencies — they are not installed with
+  `pydantic-evals` and are not part of any dependency group. Install them only in projects that
+  use these integrations.
+- Both libraries make their own LLM calls, so be prepared for extra API usage when running a
+  dataset that includes these evaluators.

docs/evals/evaluators/llm-judge.md

@@ -0,0 +1,686 @@
+# LLM Judge Deep Dive
+
+The [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] evaluator uses an LLM to assess subjective qualities of outputs based on a rubric.
+
+## When to Use LLM-as-a-Judge
+
+LLM judges are ideal for evaluating qualities that require understanding and judgment:
+
+**Good Use Cases:**
+
+- Factual accuracy
+- Helpfulness and relevance
+- Tone and style compliance
+- Completeness of responses
+- Following complex instructions
+- RAG groundedness (does the answer use provided context?)
+- Citation accuracy
+
+**Poor Use Cases:**
+
+- Format validation (use [`IsInstance`][pydantic_evals.evaluators.IsInstance] instead)
+- Exact matching (use [`EqualsExpected`][pydantic_evals.evaluators.EqualsExpected])
+- Performance checks (use [`MaxDuration`][pydantic_evals.evaluators.MaxDuration])
+- Deterministic logic (write a custom evaluator)
+
+## Basic Usage
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='factual_accuracy',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        LLMJudge(rubric='Response is factually accurate'),
+    ],
+)
+```
+
+## Configuration Options
+
+### Rubric
+
+The `rubric` is your evaluation criteria. Be specific and clear:
+
+**Bad rubrics (vague):**
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(rubric='Good response')  # Too vague
+LLMJudge(rubric='Check quality')  # What aspect of quality?
+```
+
+**Good rubrics (specific):**
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(rubric='Response directly answers the user question without hallucination')
+LLMJudge(rubric='Response uses formal, professional language appropriate for business communication')
+LLMJudge(rubric='All factual claims in the response are supported by the provided context')
+```
+
+### Including Context
+
+Control what information the judge sees:
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+# Output only (default)
+LLMJudge(rubric='Response is polite')
+
+# Output + Input
+LLMJudge(
+    rubric='Response accurately answers the input question',
+    include_input=True,
+)
+
+# Output + Input + Expected Output
+LLMJudge(
+    rubric='Response is semantically equivalent to the expected output',
+    include_input=True,
+    include_expected_output=True,
+)
+```
+
+**Example:**
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='math_check',
+    cases=[
+        Case(
+            inputs='What is 2+2?',
+            expected_output='4',
+        ),
+    ],
+    evaluators=[
+        # This judge sees: output + inputs + expected_output
+        LLMJudge(
+            rubric='Response provides the same answer as expected, possibly with explanation',
+            include_input=True,
+            include_expected_output=True,
+        ),
+    ],
+)
+```
+
+### Model Selection
+
+Choose the judge model based on cost/quality tradeoffs:
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+# Default: GPT-4o (good balance)
+LLMJudge(rubric='...')
+
+# Anthropic Claude (alternative default)
+LLMJudge(
+    rubric='...',
+    model='anthropic:claude-sonnet-4-6',
+)
+
+# Cheaper option for simple checks
+LLMJudge(
+    rubric='Response contains profanity',
+    model='openai:gpt-5-mini',
+)
+
+# Premium option for nuanced evaluation
+LLMJudge(
+    rubric='Response demonstrates deep understanding of quantum mechanics',
+    model='anthropic:claude-opus-4-5',
+)
+```
+
+### Model Settings
+
+Customize model behavior:
+
+```python
+from pydantic_ai import ModelSettings
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='...',
+    model_settings=ModelSettings(
+        temperature=0.0,  # Deterministic evaluation
+        max_tokens=100,  # Shorter responses
+    ),
+)
+```
+
+## Output Modes
+
+### Assertion Only (Default)
+
+Returns pass/fail with reason:
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(rubric='Response is accurate')
+# Returns: {'LLMJudge_pass': EvaluationReason(value=True, reason='...')}
+```
+
+In reports:
+```
+┃ Assertions ┃
+┃ ✔          ┃
+```
+
+### Score Only
+
+Returns a numeric score (0.0 to 1.0):
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='Response quality',
+    score={'include_reason': True},
+    assertion=False,
+)
+# Returns: {'LLMJudge_score': EvaluationReason(value=0.85, reason='...')}
+```
+
+In reports:
+```
+┃ Scores             ┃
+┃ LLMJudge_score: 0.85 ┃
+```
+
+### Both Score and Assertion
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='Response quality',
+    score={'include_reason': True},
+    assertion={'include_reason': True},
+)
+# Returns: {
+#     'LLMJudge_score': EvaluationReason(value=0.85, reason='...'),
+#     'LLMJudge_pass': EvaluationReason(value=True, reason='...'),
+# }
+```
+
+### Custom Names
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='Response is factually accurate',
+    assertion={
+        'evaluation_name': 'accuracy',
+        'include_reason': True,
+    },
+)
+# Returns: {'accuracy': EvaluationReason(value=True, reason='...')}
+```
+
+In reports:
+```
+┃ Assertions ┃
+┃ accuracy: ✔ ┃
+```
+
+## Practical Examples
+
+### RAG Evaluation
+
+Evaluate whether a RAG system uses provided context:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+
+@dataclass
+class RAGInput:
+    question: str
+    context: str
+
+
+dataset = Dataset(
+    name='rag_evaluation',
+    cases=[
+        Case(
+            inputs=RAGInput(
+                question='What is the capital of France?',
+                context='France is a country in Europe. Its capital is Paris.',
+            ),
+        ),
+    ],
+    evaluators=[
+        LLMJudge(
+            rubric='Response answers the question using only information from the provided context',
+            include_input=True,
+            assertion={'evaluation_name': 'grounded', 'include_reason': True},
+        ),
+        LLMJudge(
+            rubric='Response cites specific quotes or facts from the context',
+            include_input=True,
+            assertion={'evaluation_name': 'uses_citations', 'include_reason': True},
+        ),
+    ],
+)
+```
+
+### Recipe Generation with Case-Specific Rubrics
+
+This example shows how to use both dataset-level and case-specific evaluators:
+
+```python {title="recipe_evaluation.py" test="skip"}
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from pydantic_ai import Agent, format_as_xml
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import IsInstance, LLMJudge
+
+
+class CustomerOrder(BaseModel):
+    dish_name: str
+    dietary_restriction: str | None = None
+
+
+class Recipe(BaseModel):
+    ingredients: list[str]
+    steps: list[str]
+
+
+recipe_agent = Agent(
+    'openai:gpt-5-mini',
+    output_type=Recipe,
+    instructions=(
+        'Generate a recipe to cook the dish that meets the dietary restrictions.'
+    ),
+)
+
+
+async def transform_recipe(customer_order: CustomerOrder) -> Recipe:
+    r = await recipe_agent.run(format_as_xml(customer_order))
+    return r.output
+
+
+recipe_dataset = Dataset[CustomerOrder, Recipe, Any](
+    name='recipe_evaluation',
+    cases=[
+        Case(
+            name='vegetarian_recipe',
+            inputs=CustomerOrder(
+                dish_name='Spaghetti Bolognese', dietary_restriction='vegetarian'
+            ),
+            expected_output=None,
+            metadata={'focus': 'vegetarian'},
+            evaluators=(  # (1)!
+                LLMJudge(
+                    rubric='Recipe should not contain meat or animal products',
+                ),
+            ),
+        ),
+        Case(
+            name='gluten_free_recipe',
+            inputs=CustomerOrder(
+                dish_name='Chocolate Cake', dietary_restriction='gluten-free'
+            ),
+            expected_output=None,
+            metadata={'focus': 'gluten-free'},
+            evaluators=(  # (2)!
+                LLMJudge(
+                    rubric='Recipe should not contain gluten or wheat products',
+                ),
+            ),
+        ),
+    ],
+    evaluators=[  # (3)!
+        IsInstance(type_name='Recipe'),
+        LLMJudge(
+            rubric='Recipe should have clear steps and relevant ingredients',
+            include_input=True,
+            model='anthropic:claude-sonnet-4-6',
+        ),
+    ],
+)
+
+
+report = recipe_dataset.evaluate_sync(transform_recipe)
+print(report)
+"""
+     Evaluation Summary: transform_recipe
+┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━┓
+┃ Case ID            ┃ Assertions ┃ Duration ┃
+┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━┩
+│ vegetarian_recipe  │ ✔✔✔        │    38.1s │
+├────────────────────┼────────────┼──────────┤
+│ gluten_free_recipe │ ✔✔✔        │    22.4s │
+├────────────────────┼────────────┼──────────┤
+│ Averages           │ 100.0% ✔   │    30.3s │
+└────────────────────┴────────────┴──────────┘
+"""
+```
+
+1. Case-specific evaluator - only runs for the vegetarian recipe case
+2. Case-specific evaluator - only runs for the gluten-free recipe case
+3. Dataset-level evaluators - run for all cases
+
+### Multi-Aspect Evaluation
+
+Use multiple judges for different quality dimensions:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='multi_aspect',
+    cases=[Case(inputs='test')],
+    evaluators=[
+        # Accuracy
+        LLMJudge(
+            rubric='Response is factually accurate',
+            include_input=True,
+            assertion={'evaluation_name': 'accurate'},
+        ),
+
+        # Helpfulness
+        LLMJudge(
+            rubric='Response is helpful and actionable',
+            include_input=True,
+            score={'evaluation_name': 'helpfulness'},
+            assertion=False,
+        ),
+
+        # Tone
+        LLMJudge(
+            rubric='Response uses professional, respectful language',
+            assertion={'evaluation_name': 'professional_tone'},
+        ),
+
+        # Safety
+        LLMJudge(
+            rubric='Response contains no harmful, biased, or inappropriate content',
+            assertion={'evaluation_name': 'safe'},
+        ),
+    ],
+)
+```
+
+### Comparative Evaluation
+
+Compare output against expected output:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='comparative_eval',
+    cases=[
+        Case(
+            name='translation',
+            inputs='Hello world',
+            expected_output='Bonjour le monde',
+        ),
+    ],
+    evaluators=[
+        LLMJudge(
+            rubric='Response is semantically equivalent to the expected output',
+            include_input=True,
+            include_expected_output=True,
+            score={'evaluation_name': 'semantic_similarity'},
+            assertion={'evaluation_name': 'correct_meaning'},
+        ),
+    ],
+)
+```
+
+## Best Practices
+
+### 1. Be Specific in Rubrics
+
+**Bad:**
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(rubric='Good answer')
+```
+
+**Better:**
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(rubric='Response accurately answers the question without hallucinating facts')
+```
+
+**Best:**
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='''
+    Response must:
+    1. Directly answer the question asked
+    2. Use only information from the provided context
+    3. Cite specific passages from the context
+    4. Acknowledge if information is insufficient
+    ''',
+    include_input=True,
+)
+```
+
+### 2. Use Multiple Judges
+
+Don't always try to evaluate everything with one rubric:
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+
+# Instead of this:
+LLMJudge(rubric='Response is good, accurate, helpful, and safe')
+
+# Do this:
+evaluators = [
+    LLMJudge(rubric='Response is factually accurate'),
+    LLMJudge(rubric='Response is helpful and actionable'),
+    LLMJudge(rubric='Response is safe and appropriate'),
+]
+```
+
+### 3. Combine with Deterministic Checks
+
+Don't use LLM evaluation for checks that can be done deterministically:
+
+```python
+from pydantic_evals.evaluators import Contains, IsInstance, LLMJudge
+
+evaluators = [
+    IsInstance(type_name='str'),
+    Contains(value='required_section'),
+    LLMJudge(rubric='Response quality is high'),
+]
+```
+
+### 4. Use Temperature 0 for Consistency
+
+```python
+from pydantic_ai import ModelSettings
+from pydantic_evals.evaluators import LLMJudge
+
+LLMJudge(
+    rubric='...',
+    model_settings=ModelSettings(temperature=0.0),
+)
+```
+
+
+## Limitations
+
+### Non-Determinism
+
+LLM judges are not deterministic. The same output may receive different scores across runs.
+
+**Mitigation:**
+
+- Use `temperature=0.0` for more consistency
+- Run multiple evaluations and average
+- Use retry strategies for flaky evaluations
+
+### Cost
+
+LLM judges make API calls, which cost money and time.
+
+**Mitigation:**
+
+- Use cheaper models for simple checks (`gpt-5-mini`)
+- Run deterministic checks first to fail fast
+- Cache results when possible
+- Limit evaluation to changed cases
+
+### Model Biases
+
+LLM judges inherit biases from their training data.
+
+**Mitigation:**
+
+- Use multiple judge models and compare
+- Review evaluation reasons, not just scores
+- Validate judges against human-labeled test sets
+- Be aware of known biases (length bias, style preferences)
+
+### Context Limits
+
+Judges have token limits for inputs.
+
+**Mitigation:**
+
+- Truncate long inputs/outputs intelligently
+- Use focused rubrics that don't require full context
+- Consider chunked evaluation for very long content
+
+## Debugging LLM Judges
+
+### View Reasons
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+
+def my_task(inputs: str) -> str:
+    return f'Result: {inputs}'
+
+
+dataset = Dataset(
+    name='debug_reasons',
+    cases=[Case(inputs='test')],
+    evaluators=[LLMJudge(rubric='Response is clear')],
+)
+report = dataset.evaluate_sync(my_task)
+report.print(include_reasons=True)
+"""
+     Evaluation Summary: my_task
+┏━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┓
+┃ Case ID  ┃ Assertions  ┃ Duration ┃
+┡━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━┩
+│ Case 1   │ LLMJudge: ✔ │     10ms │
+│          │   Reason: - │          │
+│          │             │          │
+│          │             │          │
+├──────────┼─────────────┼──────────┤
+│ Averages │ 100.0% ✔    │     10ms │
+└──────────┴─────────────┴──────────┘
+"""
+```
+
+Output:
+```
+┃ Assertions              ┃
+┃ accuracy: ✔            ┃
+┃   Reason: The response │
+┃   correctly states...  │
+```
+
+### Access Programmatically
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+
+def my_task(inputs: str) -> str:
+    return f'Result: {inputs}'
+
+
+dataset = Dataset(
+    name='programmatic_access',
+    cases=[Case(inputs='test')],
+    evaluators=[LLMJudge(rubric='Response is clear')],
+)
+report = dataset.evaluate_sync(my_task)
+for case in report.cases:
+    for name, result in case.assertions.items():
+        print(f'{name}: {result.value}')
+        #> LLMJudge: True
+        if result.reason:
+            print(f'  Reason: {result.reason}')
+            #>   Reason: -
+```
+
+### Compare Judges
+
+Test the same cases with different judge models:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+
+def my_task(inputs: str) -> str:
+    return f'Result: {inputs}'
+
+
+judges = [
+    LLMJudge(rubric='Response is clear', model='openai:gpt-5.2'),
+    LLMJudge(rubric='Response is clear', model='anthropic:claude-sonnet-4-6'),
+    LLMJudge(rubric='Response is clear', model='openai:gpt-5-mini'),
+]
+
+for judge in judges:
+    dataset = Dataset(name='judge_comparison', cases=[Case(inputs='test')], evaluators=[judge])
+    report = dataset.evaluate_sync(my_task)
+    # Compare results
+```
+
+## Advanced: Custom Judge Models
+
+Set a default judge model for all `LLMJudge` evaluators:
+
+```python
+from pydantic_evals.evaluators import LLMJudge
+from pydantic_evals.evaluators.llm_as_a_judge import set_default_judge_model
+
+# Set default to Claude
+set_default_judge_model('anthropic:claude-sonnet-4-6')
+
+# Now all LLMJudge instances use Claude by default
+LLMJudge(rubric='...')  # Uses Claude
+```
+
+## Next Steps
+
+- **[Custom Evaluators](custom.md)** - Write custom evaluation logic
+- **[Native Evaluators](built-in.md)** - Complete evaluator reference

docs/evals/evaluators/overview.md

@@ -0,0 +1,491 @@
+# Evaluators Overview
+
+Evaluators are the core of Pydantic Evals. They analyze task outputs and provide scores, labels, or pass/fail assertions.
+
+## When to Use Different Evaluators
+
+### Deterministic Checks (Fast & Reliable)
+
+Use deterministic evaluators when you can define exact rules:
+
+| Evaluator | Use Case | Example |
+|-----------|----------|---------|
+| [`EqualsExpected`][pydantic_evals.evaluators.EqualsExpected] | Exact output match | Structured data, classification |
+| [`Equals`][pydantic_evals.evaluators.Equals] | Equals specific value | Checking for sentinel values |
+| [`Contains`][pydantic_evals.evaluators.Contains] | Substring/element check | Required keywords, PII detection |
+| [`IsInstance`][pydantic_evals.evaluators.IsInstance] | Type validation | Format validation |
+| [`MaxDuration`][pydantic_evals.evaluators.MaxDuration] | Performance threshold | SLA compliance |
+| [`HasMatchingSpan`][pydantic_evals.evaluators.HasMatchingSpan] | Behavior verification | Tool calls, code paths |
+| [`ToolCorrectness`][pydantic_evals.evaluators.ToolCorrectness] | Required tool coverage | Multiset of tool names invoked |
+| [`TrajectoryMatch`][pydantic_evals.evaluators.TrajectoryMatch] | Tool-call sequence quality | F1 against expected trajectory |
+| [`ArgumentCorrectness`][pydantic_evals.evaluators.ArgumentCorrectness] | Tool argument checks | Refund `order_id`, search query |
+| [`MaxToolCalls`][pydantic_evals.evaluators.MaxToolCalls] | Budget discipline | Tool-call budget |
+| [`MaxModelRequests`][pydantic_evals.evaluators.MaxModelRequests] | Budget discipline | Model-request budget |
+
+**Advantages:**
+
+- Fast execution (microseconds to milliseconds)
+- Deterministic results
+- No cost
+- Easy to debug
+
+**When to use:**
+
+- Format validation (JSON structure, type checking)
+- Required content checks (must contain X, must not contain Y)
+- Performance requirements (latency, token counts)
+- Behavioral checks (which tools were called, which code paths executed)
+
+### LLM-as-a-Judge (Flexible & Nuanced)
+
+Use [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] when evaluation requires understanding or judgment:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='llm_judge_example',
+    cases=[Case(inputs='What is 2+2?', expected_output='4')],
+    evaluators=[
+        LLMJudge(
+            rubric='Response is factually accurate based on the input',
+            include_input=True,
+        )
+    ],
+)
+```
+
+For metrics aligned with widely-used evaluation methods (G-Eval, the Ragas RAG metrics, GEMBA),
+see [Standard Quality Metrics](standard-quality-metrics.md): the
+[`GEval`][pydantic_evals.evaluators.GEval] evaluator plus ready-made
+[`LLMJudge`][pydantic_evals.evaluators.LLMJudge] rubrics you can copy and adapt. To plug in the
+*exact* upstream implementations of external frameworks, see
+[Third-Party Integrations](framework-integrations.md).
+
+**Advantages:**
+
+- Can evaluate subjective qualities (helpfulness, tone, creativity)
+- Understands natural language
+- Can follow complex rubrics
+- Flexible across domains
+
+**Disadvantages:**
+
+- Slower (seconds per evaluation)
+- Costs money
+- Non-deterministic
+- Can have biases
+
+**When to use:**
+
+- Factual accuracy
+- Relevance and helpfulness
+- Tone and style
+- Completeness
+- Following instructions
+- RAG quality (groundedness, citation accuracy)
+
+### Custom Evaluators
+
+Custom evaluators can be useful if you want to make use of any evaluation logic we don't provide with the framework.
+They are frequently useful for domain-specific logic:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ValidSQL(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        try:
+            import sqlparse
+            sqlparse.parse(ctx.output)
+            return True
+        except Exception:
+            return False
+```
+
+**When to use:**
+
+- Domain-specific validation (SQL syntax, regex patterns, business rules)
+- External API calls (running generated code, checking databases)
+- Complex calculations (precision/recall, BLEU scores)
+- Integration checks (does API call succeed?)
+
+## Evaluation Types
+
+!!! info "Detailed Return Types Guide"
+    For full detail about precisely what custom Evaluators may return, see [Custom Evaluator Return Types](custom.md#return-types).
+
+Evaluators essentially return three types of results:
+
+### 1. Assertions (bool)
+
+Pass/fail checks that appear as ✔ or ✗ in reports:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class HasKeyword(Evaluator):
+    keyword: str
+
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return self.keyword in ctx.output
+```
+
+**Use for:** Binary checks, quality gates, compliance requirements
+
+### 2. Scores (int or float)
+
+Numeric metrics:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ConfidenceScore(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> float:
+        # Analyze and return score
+        return 0.87  # 87% confidence
+```
+
+**Use for:** Quality metrics, ranking, A/B testing, regression tracking
+
+### 3. Labels (str)
+
+Categorical classifications:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class SentimentClassifier(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> str:
+        if 'error' in ctx.output.lower():
+            return 'error'
+        elif 'success' in ctx.output.lower():
+            return 'success'
+        return 'neutral'
+```
+
+**Use for:** Classification, error categorization, quality buckets
+
+### Multiple Results
+
+You can return multiple evaluations from a single evaluator:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class ComprehensiveCheck(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool | float | str]:
+        return {
+            'valid_format': self._check_format(ctx.output),  # bool
+            'quality_score': self._score_quality(ctx.output),  # float
+            'category': self._classify(ctx.output),  # str
+        }
+
+    def _check_format(self, output: str) -> bool:
+        return True
+
+    def _score_quality(self, output: str) -> float:
+        return 0.85
+
+    def _classify(self, output: str) -> str:
+        return 'good'
+```
+
+## Combining Evaluators
+
+Mix and match evaluators to create comprehensive evaluation suites:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import (
+    Contains,
+    IsInstance,
+    LLMJudge,
+    MaxDuration,
+)
+
+dataset = Dataset(
+    name='layered_evaluation',
+    cases=[Case(inputs='test', expected_output='result')],
+    evaluators=[
+        # Fast deterministic checks first
+        IsInstance(type_name='str'),
+        Contains(value='required_field'),
+        MaxDuration(seconds=2.0),
+        # Slower LLM checks after
+        LLMJudge(
+            rubric='Response is accurate and helpful',
+            include_input=True,
+        ),
+    ],
+)
+```
+
+## Case-specific evaluators
+
+Case-specific evaluators are one of the most powerful features for building comprehensive evaluation suites. You can attach evaluators to individual [`Case`][pydantic_evals.dataset.Case] objects that only run for those specific cases:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import IsInstance, LLMJudge
+
+dataset = Dataset(
+    name='case_specific_evaluators',
+    cases=[
+        Case(
+            name='greeting_response',
+            inputs='Say hello',
+            evaluators=[
+                # This evaluator only runs for this case
+                LLMJudge(
+                    rubric='Response is warm and friendly, uses casual tone',
+                    include_input=True,
+                ),
+            ],
+        ),
+        Case(
+            name='formal_response',
+            inputs='Write a business email',
+            evaluators=[
+                # Different requirements for this case
+                LLMJudge(
+                    rubric='Response is professional and formal, uses business language',
+                    include_input=True,
+                ),
+            ],
+        ),
+    ],
+    evaluators=[
+        # This runs for ALL cases
+        IsInstance(type_name='str'),
+    ],
+)
+```
+
+### Why Case-Specific Evaluators Matter
+
+Case-specific evaluators solve a fundamental problem with one-size-fits-all evaluation: **if you could write a single evaluator rubric that perfectly captured your requirements across all cases, you'd just incorporate that rubric into your agent's instructions**. (Note: this is less relevant in cases where you want to use a cheaper model in production and assess it using a more expensive model, but in many cases it makes sense to use the best model you can in production.)
+
+The power of case-specific evaluation comes from the nuance:
+
+- **Different cases have different requirements**: A customer support response needs empathy; a technical API response needs precision
+- **Avoid "inmates running the asylum"**: If your LLMJudge rubric is generic enough to work everywhere, your agent should already be following it
+- **Capture nuanced golden behavior**: Each case can specify exactly what "good" looks like for that scenario
+
+### Building Golden Datasets with Case-Specific LLMJudge
+
+A particularly powerful pattern is using case-specific [`LLMJudge`][pydantic_evals.evaluators.LLMJudge] evaluators to quickly build comprehensive, maintainable evaluation suites. Instead of needing exact `expected_output` values, you can describe what you care about:
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import LLMJudge
+
+dataset = Dataset(
+    name='golden_dataset',
+    cases=[
+        Case(
+            name='handle_refund_request',
+            inputs={'query': 'I want my money back', 'order_id': '12345'},
+            evaluators=[
+                LLMJudge(
+                    rubric="""
+                    Response should:
+                    1. Acknowledge the refund request empathetically
+                    2. Ask for the reason for the refund
+                    3. Mention our 30-day refund policy
+                    4. NOT process the refund immediately (needs manager approval)
+                    """,
+                    include_input=True,
+                ),
+            ],
+        ),
+        Case(
+            name='handle_shipping_question',
+            inputs={'query': 'Where is my order?', 'order_id': '12345'},
+            evaluators=[
+                LLMJudge(
+                    rubric="""
+                    Response should:
+                    1. Confirm the order number
+                    2. Provide tracking information
+                    3. Give estimated delivery date
+                    4. Be brief and factual (not overly apologetic)
+                    """,
+                    include_input=True,
+                ),
+            ],
+        ),
+        Case(
+            name='handle_angry_customer',
+            inputs={'query': 'This is completely unacceptable!', 'order_id': '12345'},
+            evaluators=[
+                LLMJudge(
+                    rubric="""
+                    Response should:
+                    1. Prioritize de-escalation with empathy
+                    2. Avoid being defensive
+                    3. Offer concrete next steps
+                    4. Use phrases like "I understand" and "Let me help"
+                    """,
+                    include_input=True,
+                ),
+            ],
+        ),
+    ],
+)
+```
+
+This approach lets you:
+
+- **Build comprehensive test suites quickly**: Just describe what you want per case
+- **Maintain easily**: Update rubrics as requirements change, without regenerating outputs
+- **Cover edge cases naturally**: Add new cases with specific requirements as you discover them
+- **Capture domain knowledge**: Each rubric documents what "good" means for that scenario
+
+The LLM evaluator excels at understanding nuanced requirements and assessing compliance, making this a practical way to create thorough evaluation coverage without brittleness.
+
+## Async vs Sync
+
+Evaluators can be sync or async:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+@dataclass
+class SyncEvaluator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        return True
+
+
+async def some_async_operation() -> bool:
+    return True
+
+
+@dataclass
+class AsyncEvaluator(Evaluator):
+    async def evaluate(self, ctx: EvaluatorContext) -> bool:
+        result = await some_async_operation()
+        return result
+```
+
+Pydantic Evals handles both automatically. Use async when:
+- Making API calls
+- Running database queries
+- Performing I/O operations
+- Calling LLMs (like [`LLMJudge`][pydantic_evals.evaluators.LLMJudge])
+
+## Evaluation Context
+
+All evaluators receive an [`EvaluatorContext`][pydantic_evals.evaluators.EvaluatorContext]:
+
+- `ctx.inputs` - Task inputs
+- `ctx.output` - Task output (to evaluate)
+- `ctx.expected_output` - Expected output (if provided)
+- `ctx.metadata` - Case metadata (if provided)
+- `ctx.duration` - Task execution time (seconds)
+- `ctx.span_tree` - OpenTelemetry spans (if logfire configured)
+- `ctx.metrics` - Custom metrics dict
+- `ctx.attributes` - Custom attributes dict
+
+This gives evaluators full context to make informed assessments.
+
+## Error Handling
+
+If an evaluator raises an exception, it's captured as an [`EvaluatorFailure`][pydantic_evals.evaluators.EvaluatorFailure]:
+
+```python
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+
+def risky_operation(output: str) -> bool:
+    # This might raise an exception
+    if 'error' in output:
+        raise ValueError('Found error in output')
+    return True
+
+
+@dataclass
+class RiskyEvaluator(Evaluator):
+    def evaluate(self, ctx: EvaluatorContext) -> bool:
+        # If this raises an exception, it will be captured
+        result = risky_operation(ctx.output)
+        return result
+```
+
+Failures appear in `report.cases[i].evaluator_failures` with:
+
+- Evaluator name
+- Error message
+- Full stacktrace
+
+Use retry configuration to handle transient failures (see [Retry Strategies](../how-to/retry-strategies.md)).
+
+## Report Evaluators (Experiment-Wide)
+
+All the evaluators above run once per case. **Report evaluators** are different: they run once per
+experiment after all cases have been evaluated, and analyze the full set of results together.
+
+Use report evaluators for experiment-wide statistics like:
+
+- **Confusion matrices** — visualize classification accuracy across classes
+- **Precision-recall curves** — assess ranking quality with AUC scores
+- **Scalar metrics** — overall accuracy, F1, BLEU, or any single number
+- **Summary tables** — per-class breakdowns, error category summaries
+
+```python
+from pydantic_evals import Case, Dataset
+from pydantic_evals.evaluators import ConfusionMatrixEvaluator
+
+dataset = Dataset(
+    name='report_evaluator_example',
+    cases=[
+        Case(inputs='meow', expected_output='cat'),
+        Case(inputs='woof', expected_output='dog'),
+    ],
+    report_evaluators=[
+        ConfusionMatrixEvaluator(
+            predicted_from='output',
+            expected_from='expected_output',
+        ),
+    ],
+)
+```
+
+**See:** [Report Evaluators](report-evaluators.md) for the full guide, including built-in report evaluators and how to write custom ones.
+
+## Next Steps
+
+- **[Native Evaluators](built-in.md)** - Complete reference of all provided evaluators
+- **[LLM Judge](llm-judge.md)** - Deep dive on LLM-as-a-Judge evaluation
+- **[Standard Quality Metrics](standard-quality-metrics.md)** - G-Eval, plus LLM judge rubrics for common RAG and translation metrics
+- **[Third-Party Integrations](framework-integrations.md)** - Wrap Ragas, DeepEval, and other metrics libraries
+- **[Custom Evaluators](custom.md)** - Write your own evaluation logic
+- **[Report Evaluators](report-evaluators.md)** - Experiment-wide analyses
+- **[Span-Based Evaluation](span-based.md)** - Evaluate using OpenTelemetry spans
+- **[Agentic Evaluators](agentic.md)** - Trajectory, tool-correctness, argument, and step-budget checks for agents