wehub-resource-sync
9201ef759e
Harness Compat / harness compat (push) Failing after 0s
CI / test on 3.12 (standard) (push) Has been cancelled
CI / test on 3.13 (standard) (push) Has been cancelled
CI / test on 3.14 (standard) (push) Has been cancelled
CI / test on 3.10 (all-extras) (push) Has been cancelled
CI / test on 3.11 (all-extras) (push) Has been cancelled
CI / test on 3.12 (all-extras) (push) Has been cancelled
CI / test on 3.14 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.10 (pydantic-evals) (push) Has been cancelled
CI / test on 3.11 (pydantic-evals) (push) Has been cancelled
CI / test on 3.12 (pydantic-evals) (push) Has been cancelled
CI / deploy-docs-preview (push) Has been cancelled
CI / build release artifacts (push) Has been cancelled
CI / publish to PyPI (push) Has been cancelled
CI / Send tweet (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / mypy (push) Has been cancelled
CI / docs (push) Has been cancelled
CI / test on 3.10 (standard) (push) Has been cancelled
CI / test on 3.11 (standard) (push) Has been cancelled
CI / test on 3.13 (all-extras) (push) Has been cancelled
CI / test on 3.14 (all-extras) (push) Has been cancelled
CI / test on 3.10 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.11 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.12 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-evals) (push) Has been cancelled
CI / test on 3.14 (pydantic-evals) (push) Has been cancelled
CI / test on 3.10 (lowest-versions) (push) Has been cancelled
CI / test on 3.11 (lowest-versions) (push) Has been cancelled
CI / test on 3.12 (lowest-versions) (push) Has been cancelled
CI / test on 3.13 (lowest-versions) (push) Has been cancelled
CI / test on 3.14 (lowest-versions) (push) Has been cancelled
CI / test examples on 3.11 (push) Has been cancelled
CI / test examples on 3.12 (push) Has been cancelled
CI / test examples on 3.13 (push) Has been cancelled
CI / test examples on 3.14 (push) Has been cancelled
CI / coverage (push) Has been cancelled
CI / check (push) Has been cancelled
CI / deploy-docs (push) Has been cancelled
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
from pathlib import Path
|
|
from types import NoneType
|
|
|
|
import logfire
|
|
|
|
from pydantic_ai_examples.evals import infer_time_range
|
|
from pydantic_ai_examples.evals.agent import time_range_agent
|
|
from pydantic_ai_examples.evals.custom_evaluators import (
|
|
CUSTOM_EVALUATOR_TYPES,
|
|
)
|
|
from pydantic_ai_examples.evals.models import (
|
|
TimeRangeInputs,
|
|
TimeRangeResponse,
|
|
)
|
|
from pydantic_evals import Dataset
|
|
|
|
logfire.configure(
|
|
send_to_logfire='if-token-present',
|
|
environment='development',
|
|
service_name='evals',
|
|
)
|
|
logfire.instrument_pydantic_ai()
|
|
|
|
|
|
def compare_models():
|
|
dataset_path = Path(__file__).parent / 'datasets' / 'time_range_v2.yaml'
|
|
dataset = Dataset[TimeRangeInputs, TimeRangeResponse, NoneType].from_file(
|
|
dataset_path, custom_evaluator_types=CUSTOM_EVALUATOR_TYPES
|
|
)
|
|
with logfire.span('Comparing different models for time_range_agent'):
|
|
with time_range_agent.override(model='openai:gpt-5.1'):
|
|
dataset.evaluate_sync(infer_time_range, name='openai:gpt-5.1')
|
|
with time_range_agent.override(model='openai:gpt-5.2'):
|
|
dataset.evaluate_sync(infer_time_range, name='openai:gpt-5.2')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
compare_models()
|