wehub-resource-sync
9201ef759e
Harness Compat / harness compat (push) Failing after 0s
CI / test on 3.12 (standard) (push) Has been cancelled
CI / test on 3.13 (standard) (push) Has been cancelled
CI / test on 3.14 (standard) (push) Has been cancelled
CI / test on 3.10 (all-extras) (push) Has been cancelled
CI / test on 3.11 (all-extras) (push) Has been cancelled
CI / test on 3.12 (all-extras) (push) Has been cancelled
CI / test on 3.14 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.10 (pydantic-evals) (push) Has been cancelled
CI / test on 3.11 (pydantic-evals) (push) Has been cancelled
CI / test on 3.12 (pydantic-evals) (push) Has been cancelled
CI / deploy-docs-preview (push) Has been cancelled
CI / build release artifacts (push) Has been cancelled
CI / publish to PyPI (push) Has been cancelled
CI / Send tweet (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / mypy (push) Has been cancelled
CI / docs (push) Has been cancelled
CI / test on 3.10 (standard) (push) Has been cancelled
CI / test on 3.11 (standard) (push) Has been cancelled
CI / test on 3.13 (all-extras) (push) Has been cancelled
CI / test on 3.14 (all-extras) (push) Has been cancelled
CI / test on 3.10 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.11 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.12 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-evals) (push) Has been cancelled
CI / test on 3.14 (pydantic-evals) (push) Has been cancelled
CI / test on 3.10 (lowest-versions) (push) Has been cancelled
CI / test on 3.11 (lowest-versions) (push) Has been cancelled
CI / test on 3.12 (lowest-versions) (push) Has been cancelled
CI / test on 3.13 (lowest-versions) (push) Has been cancelled
CI / test on 3.14 (lowest-versions) (push) Has been cancelled
CI / test examples on 3.11 (push) Has been cancelled
CI / test examples on 3.12 (push) Has been cancelled
CI / test examples on 3.13 (push) Has been cancelled
CI / test examples on 3.14 (push) Has been cancelled
CI / coverage (push) Has been cancelled
CI / check (push) Has been cancelled
CI / deploy-docs (push) Has been cancelled
43 lines
1.2 KiB
Python
43 lines
1.2 KiB
Python
from pathlib import Path
|
|
from types import NoneType
|
|
|
|
import logfire
|
|
|
|
from pydantic_ai_examples.evals import infer_time_range
|
|
from pydantic_ai_examples.evals.custom_evaluators import (
|
|
CUSTOM_EVALUATOR_TYPES,
|
|
)
|
|
from pydantic_ai_examples.evals.models import (
|
|
TimeRangeInputs,
|
|
TimeRangeResponse,
|
|
)
|
|
from pydantic_evals import Dataset
|
|
|
|
logfire.configure(
|
|
send_to_logfire='if-token-present',
|
|
environment='development',
|
|
service_name='evals',
|
|
)
|
|
logfire.instrument_pydantic_ai()
|
|
|
|
|
|
def evaluate_dataset():
|
|
dataset_path = Path(__file__).parent / 'datasets' / 'time_range_v2.yaml'
|
|
dataset = Dataset[TimeRangeInputs, TimeRangeResponse, NoneType].from_file(
|
|
dataset_path, custom_evaluator_types=CUSTOM_EVALUATOR_TYPES
|
|
)
|
|
report = dataset.evaluate_sync(infer_time_range)
|
|
print(report)
|
|
|
|
averages = report.averages()
|
|
assert averages is not None
|
|
assertion_pass_rate = averages.assertions
|
|
assert assertion_pass_rate is not None, 'There should be at least one assertion'
|
|
assert assertion_pass_rate > 0.9, (
|
|
f'The assertion pass rate was {assertion_pass_rate:.1%}; it should be above 90%.'
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
evaluate_dataset()
|