wehub-resource-sync
9201ef759e
Harness Compat / harness compat (push) Failing after 0s
CI / test on 3.12 (standard) (push) Has been cancelled
CI / test on 3.13 (standard) (push) Has been cancelled
CI / test on 3.14 (standard) (push) Has been cancelled
CI / test on 3.10 (all-extras) (push) Has been cancelled
CI / test on 3.11 (all-extras) (push) Has been cancelled
CI / test on 3.12 (all-extras) (push) Has been cancelled
CI / test on 3.14 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.10 (pydantic-evals) (push) Has been cancelled
CI / test on 3.11 (pydantic-evals) (push) Has been cancelled
CI / test on 3.12 (pydantic-evals) (push) Has been cancelled
CI / deploy-docs-preview (push) Has been cancelled
CI / build release artifacts (push) Has been cancelled
CI / publish to PyPI (push) Has been cancelled
CI / Send tweet (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / mypy (push) Has been cancelled
CI / docs (push) Has been cancelled
CI / test on 3.10 (standard) (push) Has been cancelled
CI / test on 3.11 (standard) (push) Has been cancelled
CI / test on 3.13 (all-extras) (push) Has been cancelled
CI / test on 3.14 (all-extras) (push) Has been cancelled
CI / test on 3.10 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.11 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.12 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-ai-slim) (push) Has been cancelled
CI / test on 3.13 (pydantic-evals) (push) Has been cancelled
CI / test on 3.14 (pydantic-evals) (push) Has been cancelled
CI / test on 3.10 (lowest-versions) (push) Has been cancelled
CI / test on 3.11 (lowest-versions) (push) Has been cancelled
CI / test on 3.12 (lowest-versions) (push) Has been cancelled
CI / test on 3.13 (lowest-versions) (push) Has been cancelled
CI / test on 3.14 (lowest-versions) (push) Has been cancelled
CI / test examples on 3.11 (push) Has been cancelled
CI / test examples on 3.12 (push) Has been cancelled
CI / test examples on 3.13 (push) Has been cancelled
CI / test examples on 3.14 (push) Has been cancelled
CI / coverage (push) Has been cancelled
CI / check (push) Has been cancelled
CI / deploy-docs (push) Has been cancelled
433 lines
17 KiB
Python
433 lines
17 KiB
Python
"""Tests for Google CodeExecutionTool (uses executableCode/codeExecutionResult parts, not toolCall/toolResponse)."""
|
|
|
|
from __future__ import annotations as _annotations
|
|
|
|
from datetime import timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
from inline_snapshot import snapshot
|
|
|
|
from pydantic_ai import (
|
|
Agent,
|
|
AgentStreamEvent,
|
|
FinalResultEvent,
|
|
ModelRequest,
|
|
ModelResponse,
|
|
NativeToolCallPart,
|
|
NativeToolReturnPart,
|
|
PartDeltaEvent,
|
|
PartEndEvent,
|
|
PartStartEvent,
|
|
TextPart,
|
|
TextPartDelta,
|
|
UserPromptPart,
|
|
)
|
|
from pydantic_ai.capabilities import NativeTool
|
|
from pydantic_ai.native_tools import CodeExecutionTool
|
|
from pydantic_ai.usage import RequestUsage
|
|
|
|
from ...conftest import IsDatetime, IsNow, IsStr, try_import
|
|
from ...parts_from_messages import part_types_from_messages
|
|
|
|
with try_import() as imports_successful:
|
|
from pydantic_ai.models.google import GoogleModel
|
|
from pydantic_ai.providers.google import GoogleProvider
|
|
|
|
with try_import() as anthropic_available:
|
|
from pydantic_ai.models.anthropic import AnthropicModel
|
|
from pydantic_ai.providers.anthropic import AnthropicProvider
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
|
|
GoogleModelFactory = Callable[..., GoogleModel]
|
|
|
|
pytestmark = [
|
|
pytest.mark.skipif(not imports_successful(), reason='google-genai not installed'),
|
|
pytest.mark.anyio,
|
|
pytest.mark.vcr,
|
|
pytest.mark.filterwarnings('ignore:.*is deprecated and will reach end-of-life.*:DeprecationWarning'),
|
|
]
|
|
|
|
|
|
async def test_code_execution_stream(
|
|
allow_model_requests: None,
|
|
google_model: GoogleModelFactory,
|
|
):
|
|
"""Test Gemini streaming only code execution result or executable_code."""
|
|
m = google_model('gemini-3-flash-preview')
|
|
agent = Agent(
|
|
model=m,
|
|
instructions='Be concise and always use Python to do calculations no matter how small.',
|
|
capabilities=[NativeTool(CodeExecutionTool())],
|
|
)
|
|
|
|
event_parts: list[AgentStreamEvent] = []
|
|
async with agent.iter(user_prompt='what is 65465-6544 * 65464-6+1.02255') as agent_run:
|
|
async for node in agent_run:
|
|
if Agent.is_model_request_node(node) or Agent.is_call_tools_node(node):
|
|
async with node.stream(agent_run.ctx) as request_stream:
|
|
async for event in request_stream:
|
|
event_parts.append(event)
|
|
|
|
assert agent_run.result is not None
|
|
assert agent_run.result.all_messages() == snapshot(
|
|
[
|
|
ModelRequest(
|
|
parts=[
|
|
UserPromptPart(
|
|
content='what is 65465-6544 * 65464-6+1.02255',
|
|
timestamp=IsDatetime(),
|
|
)
|
|
],
|
|
instructions='Be concise and always use Python to do calculations no matter how small.',
|
|
timestamp=IsNow(tz=timezone.utc),
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
ModelResponse(
|
|
parts=[
|
|
NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
result = 65465 - 6544 * 65464 - 6 + 1.02255
|
|
print(result)\
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': '8xju7mua',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={'outcome': 'OUTCOME_OK', 'output': '-428330955.97745\n', 'id': '8xju7mua'},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
TextPart(
|
|
content='The result of $65465 - 6544 \\times 65464 - 6 + 1.02255$ is **-428,330,955.97745**.',
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
],
|
|
usage=RequestUsage(
|
|
input_tokens=507,
|
|
output_tokens=276,
|
|
details={
|
|
'thoughts_tokens': 168,
|
|
'tool_use_prompt_tokens': 360,
|
|
'text_prompt_tokens': 147,
|
|
'text_tool_use_prompt_tokens': 360,
|
|
},
|
|
),
|
|
model_name='gemini-3-flash-preview',
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
provider_url='https://generativelanguage.googleapis.com/',
|
|
provider_details={'finish_reason': 'STOP'},
|
|
provider_response_id=IsStr(),
|
|
finish_reason='stop',
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
]
|
|
)
|
|
assert event_parts == snapshot(
|
|
[
|
|
PartStartEvent(
|
|
index=0,
|
|
part=NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
result = 65465 - 6544 * 65464 - 6 + 1.02255
|
|
print(result)\
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': '8xju7mua',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
),
|
|
PartEndEvent(
|
|
index=0,
|
|
part=NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
result = 65465 - 6544 * 65464 - 6 + 1.02255
|
|
print(result)\
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': '8xju7mua',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
next_part_kind='builtin-tool-return',
|
|
),
|
|
PartStartEvent(
|
|
index=1,
|
|
part=NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={'outcome': 'OUTCOME_OK', 'output': '-428330955.97745\n', 'id': '8xju7mua'},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
previous_part_kind='builtin-tool-call',
|
|
),
|
|
PartStartEvent(
|
|
index=2,
|
|
part=TextPart(content='The result of $65465 - 6544 \\times 6546'),
|
|
previous_part_kind='builtin-tool-return',
|
|
),
|
|
FinalResultEvent(tool_name=None, tool_call_id=None),
|
|
PartDeltaEvent(index=2, delta=TextPartDelta(content_delta='4 - 6 + 1.02255$ is **-428,330')),
|
|
PartDeltaEvent(index=2, delta=TextPartDelta(content_delta=',955.97745**.')),
|
|
PartDeltaEvent(
|
|
index=2,
|
|
delta=TextPartDelta(
|
|
content_delta='',
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
),
|
|
PartEndEvent(
|
|
index=2,
|
|
part=TextPart(
|
|
content='The result of $65465 - 6544 \\times 65464 - 6 + 1.02255$ is **-428,330,955.97745**.',
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
),
|
|
]
|
|
)
|
|
|
|
|
|
async def test_code_execution(allow_model_requests: None, google_model: GoogleModelFactory):
|
|
m = google_model('gemini-3-flash-preview')
|
|
agent = Agent(m, instructions='You are a helpful chatbot.', capabilities=[NativeTool(CodeExecutionTool())])
|
|
|
|
result = await agent.run('What day is today in Utrecht?')
|
|
assert result.all_messages() == snapshot(
|
|
[
|
|
ModelRequest(
|
|
parts=[UserPromptPart(content='What day is today in Utrecht?', timestamp=IsDatetime())],
|
|
timestamp=IsNow(tz=timezone.utc),
|
|
instructions='You are a helpful chatbot.',
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
ModelResponse(
|
|
parts=[
|
|
NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
from datetime import datetime
|
|
import pytz
|
|
|
|
# Get the current time in Utrecht, Netherlands (Europe/Amsterdam timezone)
|
|
utrecht_timezone = pytz.timezone('Europe/Amsterdam')
|
|
utrecht_time = datetime.now(utrecht_timezone)
|
|
|
|
# Format the date
|
|
today_date = utrecht_time.strftime("%A, %B %d, %Y")
|
|
print(f"Current day in Utrecht: {today_date}")
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': 'h0mwtrhs',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={
|
|
'outcome': 'OUTCOME_OK',
|
|
'output': 'Current day in Utrecht: Tuesday, May 05, 2026\n',
|
|
'id': 'h0mwtrhs',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
import datetime
|
|
print(datetime.datetime.now())
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': '7lr99y60',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={'outcome': 'OUTCOME_OK', 'output': '2026-05-05 20:40:33.367937\n', 'id': '7lr99y60'},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
TextPart(
|
|
content=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
],
|
|
usage=RequestUsage(
|
|
input_tokens=1989,
|
|
output_tokens=943,
|
|
details={
|
|
'thoughts_tokens': 773,
|
|
'tool_use_prompt_tokens': 1732,
|
|
'text_prompt_tokens': 196,
|
|
'text_tool_use_prompt_tokens': 1732,
|
|
},
|
|
),
|
|
model_name='gemini-3-flash-preview',
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
provider_url='https://generativelanguage.googleapis.com/',
|
|
provider_details={'finish_reason': 'STOP'},
|
|
provider_response_id=IsStr(),
|
|
finish_reason='stop',
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
]
|
|
)
|
|
|
|
result = await agent.run('What day is tomorrow?', message_history=result.all_messages())
|
|
assert result.new_messages() == snapshot(
|
|
[
|
|
ModelRequest(
|
|
parts=[UserPromptPart(content='What day is tomorrow?', timestamp=IsDatetime())],
|
|
timestamp=IsNow(tz=timezone.utc),
|
|
instructions='You are a helpful chatbot.',
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
ModelResponse(
|
|
parts=[
|
|
NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
import datetime
|
|
print(datetime.datetime.now())
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': 'l5m4dm9r',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={
|
|
'outcome': 'OUTCOME_OK',
|
|
'output': IsStr(),
|
|
'id': 'l5m4dm9r',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
NativeToolCallPart(
|
|
tool_name='code_execution',
|
|
args={
|
|
'code': """\
|
|
import datetime
|
|
import pytz
|
|
# Utrecht is Europe/Amsterdam
|
|
tz = pytz.timezone('Europe/Amsterdam')
|
|
now = datetime.datetime.now(tz)
|
|
print(f"Time in Utrecht: {now}")
|
|
""",
|
|
'language': 'PYTHON',
|
|
'id': 'tu0hnkbw',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
NativeToolReturnPart(
|
|
tool_name='code_execution',
|
|
content={
|
|
'outcome': 'OUTCOME_OK',
|
|
'output': 'Time in Utrecht: 2026-05-05 22:40:41.913103+02:00\n',
|
|
'id': 'tu0hnkbw',
|
|
},
|
|
tool_call_id=IsStr(),
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
),
|
|
TextPart(
|
|
content='Tomorrow in Utrecht will be **Friday, May 24, 2024**.',
|
|
provider_name='google',
|
|
provider_details={'thought_signature': IsStr()},
|
|
),
|
|
],
|
|
usage=RequestUsage(
|
|
input_tokens=3949,
|
|
output_tokens=1418,
|
|
details={
|
|
'thoughts_tokens': 1312,
|
|
'tool_use_prompt_tokens': 3056,
|
|
'text_prompt_tokens': 786,
|
|
'text_tool_use_prompt_tokens': 3056,
|
|
},
|
|
),
|
|
model_name='gemini-3-flash-preview',
|
|
timestamp=IsDatetime(),
|
|
provider_name='google',
|
|
provider_url='https://generativelanguage.googleapis.com/',
|
|
provider_details={'finish_reason': 'STOP'},
|
|
provider_response_id=IsStr(),
|
|
finish_reason='stop',
|
|
run_id=IsStr(),
|
|
conversation_id=IsStr(),
|
|
),
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(not anthropic_available(), reason='anthropic not installed')
|
|
async def test_receive_history_from_another_provider(
|
|
allow_model_requests: None, anthropic_api_key: str, gemini_api_key: str
|
|
):
|
|
anthropic_model = AnthropicModel('claude-sonnet-4-0', provider=AnthropicProvider(api_key=anthropic_api_key))
|
|
google_model = GoogleModel('gemini-3-flash-preview', provider=GoogleProvider(api_key=gemini_api_key))
|
|
agent = Agent(capabilities=[NativeTool(CodeExecutionTool())])
|
|
|
|
result = await agent.run('How much is 3 * 12390?', model=anthropic_model)
|
|
assert part_types_from_messages(result.all_messages()) == snapshot(
|
|
[[UserPromptPart], [TextPart, NativeToolCallPart, NativeToolReturnPart, TextPart]]
|
|
)
|
|
|
|
result = await agent.run('Multiplied by 12390', model=google_model, message_history=result.all_messages())
|
|
assert part_types_from_messages(result.all_messages()) == snapshot(
|
|
[
|
|
[UserPromptPart],
|
|
[TextPart, NativeToolCallPart, NativeToolReturnPart, TextPart],
|
|
[UserPromptPart],
|
|
[NativeToolCallPart, NativeToolReturnPart, NativeToolCallPart, NativeToolReturnPart, TextPart],
|
|
]
|
|
)
|