chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Conftest for golden tests — ensures parent test dir is importable."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def pytest_configure() -> None:
|
||||
"""Ensure parent test directory is on sys.path for helper module imports."""
|
||||
parent_test_dir = str(Path(__file__).resolve().parent.parent)
|
||||
if parent_test_dir not in sys.path:
|
||||
sys.path.insert(0, parent_test_dir)
|
||||
@@ -0,0 +1,140 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the basic agentic chat scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
return AgentFrameworkAgent(agent=stub, **kwargs)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
BASIC_PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-chat",
|
||||
"run_id": "run-chat",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
}
|
||||
|
||||
|
||||
def _text_update(text: str) -> AgentResponseUpdate:
|
||||
return AgentResponseUpdate(contents=[Content.from_text(text=text)], role="assistant")
|
||||
|
||||
|
||||
def _snapshot_role(msg: Any) -> str:
|
||||
"""Extract role string from a snapshot message (Pydantic model or dict)."""
|
||||
role = getattr(msg, "role", None) or (msg.get("role") if isinstance(msg, dict) else None)
|
||||
if role is None:
|
||||
return ""
|
||||
return str(getattr(role, "value", role))
|
||||
|
||||
|
||||
def _snapshot_content(msg: Any) -> str:
|
||||
"""Extract content string from a snapshot message."""
|
||||
content = getattr(msg, "content", None) or (msg.get("content") if isinstance(msg, dict) else "")
|
||||
return str(content) if content else ""
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_basic_chat_golden_event_sequence() -> None:
|
||||
"""Assert the exact event type sequence for a single text response."""
|
||||
agent = _build_agent([_text_update("Hi there!")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
|
||||
stream.assert_strict_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"TEXT_MESSAGE_START",
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"TEXT_MESSAGE_END",
|
||||
"MESSAGES_SNAPSHOT",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def test_basic_chat_bookends() -> None:
|
||||
"""RUN_STARTED is first, RUN_FINISHED is last."""
|
||||
agent = _build_agent([_text_update("reply")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
stream.assert_bookends()
|
||||
|
||||
|
||||
async def test_basic_chat_text_messages_balanced() -> None:
|
||||
"""Every TEXT_MESSAGE_START has a matching TEXT_MESSAGE_END."""
|
||||
agent = _build_agent([_text_update("reply")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
|
||||
async def test_basic_chat_no_errors() -> None:
|
||||
"""No RUN_ERROR events in a normal flow."""
|
||||
agent = _build_agent([_text_update("reply")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
stream.assert_no_run_error()
|
||||
|
||||
|
||||
async def test_basic_chat_message_id_consistency() -> None:
|
||||
"""All text events reference the same message_id."""
|
||||
agent = _build_agent([_text_update("reply")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
|
||||
start = stream.first("TEXT_MESSAGE_START")
|
||||
content = stream.first("TEXT_MESSAGE_CONTENT")
|
||||
end = stream.first("TEXT_MESSAGE_END")
|
||||
assert start.message_id == content.message_id == end.message_id
|
||||
|
||||
|
||||
async def test_multi_chunk_text_golden_sequence() -> None:
|
||||
"""Streaming multiple chunks produces START + multiple CONTENT + END."""
|
||||
agent = _build_agent([_text_update("Hello "), _text_update("world!")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
|
||||
stream.assert_strict_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"TEXT_MESSAGE_START",
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"TEXT_MESSAGE_END",
|
||||
"MESSAGES_SNAPSHOT",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
stream.assert_text_messages_balanced()
|
||||
stream.assert_message_ids_consistent()
|
||||
|
||||
|
||||
async def test_messages_snapshot_contains_assistant_reply() -> None:
|
||||
"""MessagesSnapshotEvent includes the assistant's accumulated text."""
|
||||
agent = _build_agent([_text_update("Hello there")])
|
||||
stream = await _run(agent, BASIC_PAYLOAD)
|
||||
|
||||
snapshot = stream.messages_snapshot()
|
||||
assistant_msgs = [m for m in snapshot if _snapshot_role(m) == "assistant"]
|
||||
assert assistant_msgs, "No assistant message in snapshot"
|
||||
assert any("Hello there" in _snapshot_content(m) for m in assistant_msgs)
|
||||
|
||||
|
||||
async def test_empty_messages_produces_start_and_finish() -> None:
|
||||
"""Empty message list still produces RUN_STARTED and RUN_FINISHED."""
|
||||
agent = _build_agent([_text_update("reply")])
|
||||
payload = {"thread_id": "t1", "run_id": "r1", "messages": []}
|
||||
stream = await _run(agent, payload)
|
||||
|
||||
stream.assert_bookends()
|
||||
assert "TEXT_MESSAGE_START" not in stream.types()
|
||||
@@ -0,0 +1,236 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the backend (server-side) tools scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
return AgentFrameworkAgent(agent=stub, **kwargs)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-tools",
|
||||
"run_id": "run-tools",
|
||||
"messages": [{"role": "user", "content": "What's the weather?"}],
|
||||
}
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_tool_call_lifecycle_golden_sequence() -> None:
|
||||
"""Assert the full event sequence for a tool call → result → text response."""
|
||||
updates = [
|
||||
# LLM calls the tool
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
|
||||
role="assistant",
|
||||
),
|
||||
# Tool result comes back
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F and sunny")],
|
||||
role="assistant",
|
||||
),
|
||||
# LLM responds with text
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's 72°F and sunny in SF!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"TEXT_MESSAGE_START", # Synthetic start for tool-only message
|
||||
"TOOL_CALL_START",
|
||||
"TOOL_CALL_ARGS",
|
||||
"TOOL_CALL_END",
|
||||
"TOOL_CALL_RESULT",
|
||||
"TEXT_MESSAGE_END", # End of synthetic message
|
||||
"TEXT_MESSAGE_START", # New message for text response
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"TEXT_MESSAGE_END",
|
||||
"MESSAGES_SNAPSHOT",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def test_tool_calls_balanced() -> None:
|
||||
"""Every TOOL_CALL_START has a matching TOOL_CALL_END."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's 72°F!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
|
||||
async def test_text_messages_balanced_with_tools() -> None:
|
||||
"""Text messages are properly balanced even around tool calls."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's 72°F!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
|
||||
async def test_tool_call_id_matches_result() -> None:
|
||||
"""TOOL_CALL_START and TOOL_CALL_RESULT reference the same tool_call_id."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
start = stream.first("TOOL_CALL_START")
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert start.tool_call_id == result.tool_call_id == "call-1"
|
||||
|
||||
|
||||
async def test_tool_result_content_preserved() -> None:
|
||||
"""TOOL_CALL_RESULT event carries the tool's result content."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F and sunny")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == "72°F and sunny"
|
||||
|
||||
|
||||
async def test_no_run_error_on_tool_flow() -> None:
|
||||
"""Tool call flow doesn't produce RUN_ERROR."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_bookends()
|
||||
|
||||
|
||||
async def test_multiple_sequential_tool_calls() -> None:
|
||||
"""Multiple sequential tool calls each produce balanced START/END pairs."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="tool_a", call_id="call-a", arguments="{}")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-a", result="result-a")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="tool_b", call_id="call-b", arguments="{}")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-b", result="result-b")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="Done!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_tool_calls_balanced()
|
||||
stream.assert_text_messages_balanced()
|
||||
stream.assert_bookends()
|
||||
|
||||
# Both tool calls should appear
|
||||
starts = stream.get("TOOL_CALL_START")
|
||||
assert len(starts) == 2
|
||||
assert {s.tool_call_name for s in starts} == {"tool_a", "tool_b"}
|
||||
|
||||
|
||||
async def test_messages_snapshot_includes_tool_calls() -> None:
|
||||
"""MessagesSnapshotEvent includes tool call and result messages."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city":"SF"}')],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's warm!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_has_type("MESSAGES_SNAPSHOT")
|
||||
snapshot = stream.messages_snapshot()
|
||||
# Should have: user message, assistant with tool_calls, tool result, assistant text
|
||||
assert len(snapshot) >= 3
|
||||
@@ -0,0 +1,364 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the deterministic tool-driven state scenario.
|
||||
|
||||
Covers issue https://github.com/microsoft/agent-framework/issues/3167 — a tool
|
||||
returning :func:`agent_framework_ag_ui.state_update` must push a deterministic
|
||||
``StateSnapshotEvent`` derived from its actual return value, orthogonal to the
|
||||
optimistic ``predict_state_config`` path. These golden tests pin the user-visible
|
||||
event stream so additive changes cannot silently regress it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent, state_update
|
||||
|
||||
STATE_SCHEMA = {
|
||||
"weather": {"type": "object", "description": "Last fetched weather"},
|
||||
}
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
kwargs.setdefault("state_schema", STATE_SCHEMA)
|
||||
return AgentFrameworkAgent(agent=stub, **kwargs)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-det-state",
|
||||
"run_id": "run-det-state",
|
||||
"messages": [{"role": "user", "content": "What's the weather in SF?"}],
|
||||
"state": {"weather": {}},
|
||||
}
|
||||
|
||||
|
||||
def _tool_call(call_id: str, name: str, arguments: str) -> AgentResponseUpdate:
|
||||
return AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name=name, call_id=call_id, arguments=arguments)],
|
||||
role="assistant",
|
||||
)
|
||||
|
||||
|
||||
def _tool_result_with_state(call_id: str, text: str, state: dict[str, Any]) -> AgentResponseUpdate:
|
||||
"""Build a function_result update whose inner item carries a state marker.
|
||||
|
||||
This mirrors what the core framework produces when a real ``@tool`` returns
|
||||
:func:`state_update`: ``parse_result`` keeps the ``Content`` as-is, and
|
||||
``Content.from_function_result`` preserves its ``additional_properties``
|
||||
inside ``items``.
|
||||
"""
|
||||
return AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_result(
|
||||
call_id=call_id,
|
||||
result=[state_update(text=text, state=state)],
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
)
|
||||
|
||||
|
||||
def _tool_result_with_display(call_id: str, text: str, tool_result: Any, **kwargs: Any) -> AgentResponseUpdate:
|
||||
"""Build a function_result update carrying an optional UI display marker."""
|
||||
return AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_result(
|
||||
call_id=call_id,
|
||||
result=[state_update(text=text, tool_result=tool_result, **kwargs)],
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
)
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_deterministic_state_emits_snapshot_after_tool_result() -> None:
|
||||
"""The happy path: STATE_SNAPSHOT follows TOOL_CALL_RESULT in order."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
state={"weather": {"city": "SF", "temp": 14, "conditions": "foggy"}},
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's 14°C and foggy in SF.")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
# Ordered subsequence: the deterministic STATE_SNAPSHOT must follow the
|
||||
# TOOL_CALL_RESULT. This is the central contract for #3167.
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"TOOL_CALL_START",
|
||||
"TOOL_CALL_ARGS",
|
||||
"TOOL_CALL_END",
|
||||
"TOOL_CALL_RESULT",
|
||||
"STATE_SNAPSHOT",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
# The final STATE_SNAPSHOT must carry the tool-driven state.
|
||||
snapshot = stream.snapshot()
|
||||
assert snapshot["weather"] == {"city": "SF", "temp": 14, "conditions": "foggy"}
|
||||
|
||||
|
||||
async def test_deterministic_state_does_not_fire_for_plain_tool_result() -> None:
|
||||
"""Regression guard: tools returning plain strings must NOT emit a new STATE_SNAPSHOT.
|
||||
|
||||
The initial STATE_SNAPSHOT fires once from the schema + initial payload
|
||||
state. A plain (non-state_update) tool result must not add another one.
|
||||
"""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_result(call_id="call-1", result="14°C foggy")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="It's 14°C and foggy.")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
snapshots = stream.get("STATE_SNAPSHOT")
|
||||
# Only the initial snapshot (from state_schema + payload state) should exist.
|
||||
# No deterministic snapshot should have been added by the plain tool result.
|
||||
assert len(snapshots) == 1, (
|
||||
f"Expected exactly 1 STATE_SNAPSHOT (initial only) for plain tool result; "
|
||||
f"got {len(snapshots)}. Snapshots: {[s.snapshot for s in snapshots]}"
|
||||
)
|
||||
|
||||
|
||||
async def test_deterministic_state_merges_into_initial_state() -> None:
|
||||
"""The tool-driven snapshot must merge into, not replace, pre-existing state keys."""
|
||||
payload = dict(PAYLOAD)
|
||||
payload["state"] = {"weather": {}, "user_preferences": {"unit": "C"}}
|
||||
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Weather: 14°C",
|
||||
state={"weather": {"city": "SF", "temp": 14}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates, state_schema={**STATE_SCHEMA, "user_preferences": {"type": "object"}})
|
||||
stream = await _run(agent, payload)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
final_snapshot = stream.snapshot()
|
||||
assert final_snapshot["weather"] == {"city": "SF", "temp": 14}
|
||||
assert final_snapshot["user_preferences"] == {"unit": "C"}, (
|
||||
"Pre-existing state keys must survive the deterministic merge"
|
||||
)
|
||||
|
||||
|
||||
async def test_deterministic_state_llm_visible_text_is_clean() -> None:
|
||||
"""The LLM-visible TOOL_CALL_RESULT content must not leak the state marker key."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
state={"weather": {"city": "SF", "temp": 14}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == "Weather in SF: 14°C foggy"
|
||||
# The marker key must never appear in the content sent back to the LLM.
|
||||
assert "__ag_ui_tool_result_state__" not in result.content
|
||||
assert "weather" not in result.content # not as a raw state dump
|
||||
|
||||
|
||||
async def test_deterministic_state_multiple_tools_merge_in_order() -> None:
|
||||
"""Two state-updating tools in one run merge in order; later wins on key collisions."""
|
||||
updates = [
|
||||
_tool_call("call-a", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-a",
|
||||
text="First result",
|
||||
state={"weather": {"city": "SF", "temp": 14}, "source": "primary"},
|
||||
),
|
||||
_tool_call("call-b", "get_weather_refined", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-b",
|
||||
text="Refined result",
|
||||
state={"source": "refined"},
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="Here you go.")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(
|
||||
updates,
|
||||
state_schema={**STATE_SCHEMA, "source": {"type": "string"}},
|
||||
)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_tool_calls_balanced()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# Two tool-driven snapshots emitted (one per tool) plus the initial snapshot.
|
||||
snapshots = stream.get("STATE_SNAPSHOT")
|
||||
assert len(snapshots) >= 2, f"Expected at least 2 STATE_SNAPSHOTs; got {len(snapshots)}"
|
||||
|
||||
final = stream.snapshot()
|
||||
assert final["weather"] == {"city": "SF", "temp": 14}
|
||||
# Later tool must override earlier tool on the shared key.
|
||||
assert final["source"] == "refined"
|
||||
|
||||
|
||||
async def test_deterministic_state_coexists_with_predict_state_config() -> None:
|
||||
"""Predictive state and deterministic state must coexist without clobbering each other."""
|
||||
predict_config = {
|
||||
"draft": {
|
||||
"tool": "write_draft",
|
||||
"tool_argument": "body",
|
||||
}
|
||||
}
|
||||
updates = [
|
||||
# Predictive tool: its argument "body" populates state.draft optimistically.
|
||||
_tool_call("call-1", "write_draft", '{"body": "Hello world"}'),
|
||||
# Then a deterministic tool result landing a different key.
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Draft saved",
|
||||
state={"weather": {"city": "SF", "temp": 14}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(
|
||||
updates,
|
||||
state_schema={**STATE_SCHEMA, "draft": {"type": "string"}},
|
||||
predict_state_config=predict_config,
|
||||
require_confirmation=False,
|
||||
)
|
||||
payload = dict(PAYLOAD)
|
||||
payload["state"] = {"weather": {}, "draft": ""}
|
||||
stream = await _run(agent, payload)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
# The final observed state must contain both the deterministic and predictive contributions.
|
||||
final = stream.snapshot()
|
||||
assert final["weather"] == {"city": "SF", "temp": 14}, f"Deterministic state missing from final snapshot: {final}"
|
||||
|
||||
|
||||
async def test_tool_result_display_payload_reaches_ui_event_only() -> None:
|
||||
"""Rich display payload overrides TOOL_CALL_RESULT without leaking marker keys."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_display(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
|
||||
assert "__ag_ui_tool_result_display__" not in result.content
|
||||
assert "__ag_ui_tool_result_state__" not in result.content
|
||||
|
||||
|
||||
async def test_tool_result_display_falls_back_to_text_when_unset() -> None:
|
||||
"""Without a display marker, the UI event keeps the existing text content."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
state={"weather": {"city": "SF", "temp": 14}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == "Weather in SF: 14°C foggy"
|
||||
assert "__ag_ui_tool_result_display__" not in result.content
|
||||
assert "__ag_ui_tool_result_state__" not in result.content
|
||||
|
||||
|
||||
async def test_tool_result_display_coexists_with_state_snapshot() -> None:
|
||||
"""Display and durable state markers produce one deterministic state snapshot."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_display(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
|
||||
state={"weather": {"city": "SF", "temp": 14, "conditions": "foggy"}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
stream.assert_ordered_types(["TOOL_CALL_RESULT", "STATE_SNAPSHOT", "RUN_FINISHED"])
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
|
||||
|
||||
result_idx = stream.events.index(result)
|
||||
deterministic_snapshots = [
|
||||
event
|
||||
for event in stream.events[result_idx + 1 :]
|
||||
if getattr(getattr(event, "type", None), "value", getattr(event, "type", None)) == "STATE_SNAPSHOT"
|
||||
]
|
||||
assert len(deterministic_snapshots) == 1
|
||||
assert deterministic_snapshots[0].snapshot["weather"] == {
|
||||
"city": "SF",
|
||||
"temp": 14,
|
||||
"conditions": "foggy",
|
||||
}
|
||||
assert "__ag_ui_tool_result_display__" not in str(deterministic_snapshots[0].snapshot)
|
||||
assert "__ag_ui_tool_result_state__" not in str(deterministic_snapshots[0].snapshot)
|
||||
@@ -0,0 +1,90 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the generative UI (workflow-as-agent) scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import WorkflowBuilder, WorkflowContext, executor
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkWorkflow
|
||||
|
||||
|
||||
async def _run(wrapper: AgentFrameworkWorkflow, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in wrapper.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-gen-ui-agent",
|
||||
"run_id": "run-gen-ui-agent",
|
||||
"messages": [{"role": "user", "content": "Generate a UI"}],
|
||||
}
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_workflow_agent_golden_sequence() -> None:
|
||||
"""Workflow-as-agent: emits step events and text content."""
|
||||
|
||||
@executor(id="generator")
|
||||
async def generator(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("Here is your generated UI content!")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=generator).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
# Should have step events for the executor
|
||||
stream.assert_has_type("STEP_STARTED")
|
||||
stream.assert_has_type("STEP_FINISHED")
|
||||
|
||||
# Should have text message content
|
||||
stream.assert_has_type("TEXT_MESSAGE_CONTENT")
|
||||
|
||||
|
||||
async def test_workflow_agent_step_names_match() -> None:
|
||||
"""Step started/finished events reference the executor name."""
|
||||
|
||||
@executor(id="my_executor")
|
||||
async def my_executor(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("Done!")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=my_executor).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, PAYLOAD)
|
||||
|
||||
started = [e for e in stream.get("STEP_STARTED") if getattr(e, "step_name", "") == "my_executor"]
|
||||
finished = [e for e in stream.get("STEP_FINISHED") if getattr(e, "step_name", "") == "my_executor"]
|
||||
assert started, "Expected STEP_STARTED for 'my_executor'"
|
||||
assert finished, "Expected STEP_FINISHED for 'my_executor'"
|
||||
|
||||
|
||||
async def test_workflow_agent_ordered_events() -> None:
|
||||
"""Workflow events follow expected ordering: RUN_STARTED → STEP_STARTED → content → STEP_FINISHED → RUN_FINISHED."""
|
||||
|
||||
@executor(id="my_step")
|
||||
async def my_step(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("Generated content")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=my_step).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, PAYLOAD)
|
||||
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"STEP_STARTED",
|
||||
"TEXT_MESSAGE_START",
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"STEP_FINISHED",
|
||||
"TEXT_MESSAGE_END",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,135 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the client-side (declaration-only) tools scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
return AgentFrameworkAgent(agent=stub, **kwargs)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-gen-ui-tool",
|
||||
"run_id": "run-gen-ui-tool",
|
||||
"messages": [{"role": "user", "content": "Show me a chart"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "render_chart",
|
||||
"description": "Render a chart in the UI",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"data": {"type": "array"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_declaration_only_tool_golden_sequence() -> None:
|
||||
"""Declaration-only tool: TOOL_CALL_START/ARGS emitted, TOOL_CALL_END at stream end."""
|
||||
# The LLM calls a client-side tool (no server-side execution)
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="render_chart",
|
||||
call_id="call-chart",
|
||||
arguments='{"data": [1, 2, 3]}',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# Tool call start and args should be present
|
||||
stream.assert_has_type("TOOL_CALL_START")
|
||||
stream.assert_has_type("TOOL_CALL_ARGS")
|
||||
|
||||
# TOOL_CALL_END should be emitted (via get_pending_without_end)
|
||||
stream.assert_has_type("TOOL_CALL_END")
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
|
||||
async def test_declaration_only_tool_no_tool_call_result() -> None:
|
||||
"""Declaration-only tools should NOT produce TOOL_CALL_RESULT events."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="render_chart",
|
||||
call_id="call-chart",
|
||||
arguments='{"data": [1, 2, 3]}',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
assert "TOOL_CALL_RESULT" not in stream.types(), "Declaration-only tools should not have TOOL_CALL_RESULT"
|
||||
|
||||
|
||||
async def test_declaration_only_tool_text_messages_balanced() -> None:
|
||||
"""Text messages remain balanced even with declaration-only tools."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="render_chart",
|
||||
call_id="call-chart",
|
||||
arguments='{"data": [1, 2, 3]}',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
|
||||
async def test_declaration_only_tool_messages_snapshot() -> None:
|
||||
"""MessagesSnapshotEvent includes the tool call for declaration-only tools."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="render_chart",
|
||||
call_id="call-chart",
|
||||
arguments='{"data": [1, 2, 3]}',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_has_type("MESSAGES_SNAPSHOT")
|
||||
@@ -0,0 +1,194 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the HITL (human-in-the-loop) approval scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
PREDICT_CONFIG = {
|
||||
"tasks": {
|
||||
"tool": "generate_task_steps",
|
||||
"tool_argument": "steps",
|
||||
}
|
||||
}
|
||||
|
||||
STATE_SCHEMA = {
|
||||
"tasks": {"type": "array", "items": {"type": "object"}},
|
||||
}
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
return AgentFrameworkAgent(
|
||||
agent=stub,
|
||||
state_schema=STATE_SCHEMA,
|
||||
predict_state_config=PREDICT_CONFIG,
|
||||
require_confirmation=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
STEPS = [
|
||||
{"description": "Step 1: Plan", "status": "enabled"},
|
||||
{"description": "Step 2: Execute", "status": "enabled"},
|
||||
]
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-hitl",
|
||||
"run_id": "run-hitl",
|
||||
"messages": [{"role": "user", "content": "Plan my tasks"}],
|
||||
"state": {"tasks": []},
|
||||
}
|
||||
|
||||
|
||||
# ── Turn 1: Tool call → confirm_changes → interrupt ──
|
||||
|
||||
|
||||
async def test_hitl_turn1_golden_sequence() -> None:
|
||||
"""Turn 1 emits tool call, confirm_changes, and finishes with interrupt."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="generate_task_steps",
|
||||
call_id="call-steps",
|
||||
arguments=json.dumps({"steps": STEPS}),
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
# Should have: tool call start/args/end for the primary tool,
|
||||
# then TOOL_CALL_END, STATE_SNAPSHOT, confirm_changes cycle
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# confirm_changes tool call should be present
|
||||
tool_starts = stream.get("TOOL_CALL_START")
|
||||
tool_names = [getattr(s, "tool_call_name", None) for s in tool_starts]
|
||||
assert "generate_task_steps" in tool_names
|
||||
assert "confirm_changes" in tool_names
|
||||
|
||||
# RUN_FINISHED should have interrupt metadata
|
||||
interrupt = stream.run_finished_interrupts()
|
||||
assert len(interrupt) > 0
|
||||
|
||||
|
||||
async def test_hitl_turn1_tool_calls_balanced() -> None:
|
||||
"""All tool calls in turn 1 (primary + confirm_changes) are balanced."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="generate_task_steps",
|
||||
call_id="call-steps",
|
||||
arguments=json.dumps({"steps": STEPS}),
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
|
||||
async def test_hitl_turn1_text_messages_balanced() -> None:
|
||||
"""Text messages are balanced even in the approval flow."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="generate_task_steps",
|
||||
call_id="call-steps",
|
||||
arguments=json.dumps({"steps": STEPS}),
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
|
||||
# ── Turn 2: Resume with approval → confirmation message → no interrupt ──
|
||||
|
||||
|
||||
async def test_hitl_turn2_resume_with_approval() -> None:
|
||||
"""Resuming with confirm_changes result emits confirmation text and finishes cleanly."""
|
||||
# Turn 2: user sends confirm_changes result as resume
|
||||
# The agent wrapper sees a confirm_changes response and emits a confirmation message
|
||||
confirm_result = json.dumps(
|
||||
{
|
||||
"accepted": True,
|
||||
"steps": STEPS,
|
||||
}
|
||||
)
|
||||
|
||||
# Build payload with resume containing the approval
|
||||
# For confirm_changes, the messages should include the tool result
|
||||
payload: dict[str, Any] = {
|
||||
"thread_id": "thread-hitl",
|
||||
"run_id": "run-hitl-2",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Plan my tasks"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "confirm-id-1",
|
||||
"type": "function",
|
||||
"function": {"name": "confirm_changes", "arguments": json.dumps({"steps": STEPS})},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"toolCallId": "confirm-id-1",
|
||||
"content": confirm_result,
|
||||
},
|
||||
],
|
||||
"state": {"tasks": []},
|
||||
}
|
||||
|
||||
# In turn 2, the agent sees the confirm_changes result and emits a confirmation text
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text="Tasks confirmed!")],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, payload)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_text_messages_balanced()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# Should have text message content (the confirmation message)
|
||||
text_events = stream.get("TEXT_MESSAGE_CONTENT")
|
||||
assert text_events, "Expected confirmation text message"
|
||||
|
||||
# RUN_FINISHED should NOT have interrupt (approval completed)
|
||||
finished = stream.last("RUN_FINISHED")
|
||||
dumped = finished.model_dump(by_alias=True, exclude_none=True)
|
||||
assert "outcome" not in dumped, f"Expected no interrupt after approval, got {dumped.get('outcome')}"
|
||||
@@ -0,0 +1,130 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the predictive state scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
PREDICT_CONFIG = {
|
||||
"document": {
|
||||
"tool": "update_document",
|
||||
"tool_argument": "content",
|
||||
}
|
||||
}
|
||||
|
||||
STATE_SCHEMA = {
|
||||
"document": {"type": "string"},
|
||||
}
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(updates=updates)
|
||||
return AgentFrameworkAgent(
|
||||
agent=stub,
|
||||
state_schema=STATE_SCHEMA,
|
||||
predict_state_config=PREDICT_CONFIG,
|
||||
require_confirmation=False,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-predict",
|
||||
"run_id": "run-predict",
|
||||
"messages": [{"role": "user", "content": "Write a document"}],
|
||||
"state": {"document": ""},
|
||||
}
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_predictive_state_emits_deltas_during_tool_args() -> None:
|
||||
"""STATE_DELTA events are emitted as tool arguments stream in."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="update_document", call_id="call-1", arguments="")],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(name="update_document", call_id="call-1", arguments='{"content": "Hello')
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_function_call(name="update_document", call_id="call-1", arguments=' world"}')],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# PredictState custom event should be present
|
||||
custom_events = stream.get("CUSTOM")
|
||||
predict_events = [e for e in custom_events if getattr(e, "name", None) == "PredictState"]
|
||||
assert predict_events, "Expected PredictState custom event"
|
||||
|
||||
# STATE_DELTA events should be emitted during tool arg streaming
|
||||
assert "STATE_DELTA" in stream.types(), "Expected STATE_DELTA events during predictive streaming"
|
||||
|
||||
|
||||
async def test_predictive_state_snapshot_after_tool_end() -> None:
|
||||
"""STATE_SNAPSHOT is emitted when a predictive tool completes (no confirmation)."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(
|
||||
name="update_document", call_id="call-1", arguments='{"content": "Final text"}'
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
|
||||
# Should have initial state snapshot + updated snapshot after tool completion
|
||||
snapshots = stream.get("STATE_SNAPSHOT")
|
||||
assert len(snapshots) >= 1, "Expected at least one STATE_SNAPSHOT"
|
||||
|
||||
|
||||
async def test_predictive_state_ordered_events() -> None:
|
||||
"""Event ordering: RUN_STARTED → PredictState → STATE_SNAPSHOT → TOOL_CALL_* → STATE_SNAPSHOT → RUN_FINISHED."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_call(name="update_document", call_id="call-1", arguments='{"content": "doc"}')
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"CUSTOM", # PredictState
|
||||
"STATE_SNAPSHOT", # Initial state
|
||||
"TOOL_CALL_START",
|
||||
"TOOL_CALL_ARGS",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the shared state (structured output) scenario."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import AgentResponseUpdate, Content
|
||||
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkAgent
|
||||
|
||||
|
||||
class RecipeState(BaseModel):
|
||||
recipe_title: str = ""
|
||||
ingredients: list[str] = []
|
||||
message: str = ""
|
||||
|
||||
|
||||
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
|
||||
stub = StubAgent(
|
||||
updates=updates,
|
||||
default_options={"tools": None, "response_format": RecipeState},
|
||||
)
|
||||
return AgentFrameworkAgent(
|
||||
agent=stub,
|
||||
state_schema={
|
||||
"recipe_title": {"type": "string"},
|
||||
"ingredients": {"type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
PAYLOAD: dict[str, Any] = {
|
||||
"thread_id": "thread-state",
|
||||
"run_id": "run-state",
|
||||
"messages": [{"role": "user", "content": "Give me a pasta recipe"}],
|
||||
"state": {"recipe_title": "", "ingredients": []},
|
||||
}
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
async def test_shared_state_emits_state_snapshot() -> None:
|
||||
"""Structured output agent emits STATE_SNAPSHOT with parsed model fields."""
|
||||
# The structured output agent gets a response that the framework parses as RecipeState
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_text(
|
||||
text='{"recipe_title": "Pasta Carbonara", "ingredients": ["pasta", "eggs", "cheese"], "message": "Here is your recipe!"}'
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# Should have STATE_SNAPSHOT with the initial state at minimum
|
||||
stream.assert_has_type("STATE_SNAPSHOT")
|
||||
|
||||
|
||||
async def test_shared_state_initial_snapshot_on_first_update() -> None:
|
||||
"""When state_schema and state are provided, initial STATE_SNAPSHOT is emitted after RUN_STARTED."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[Content.from_text(text='{"recipe_title": "Test", "ingredients": [], "message": "hi"}')],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
# RUN_STARTED should be followed by STATE_SNAPSHOT (initial state)
|
||||
stream.assert_ordered_types(["RUN_STARTED", "STATE_SNAPSHOT"])
|
||||
|
||||
|
||||
async def test_shared_state_text_emitted_from_message_field() -> None:
|
||||
"""Structured output's 'message' field is emitted as text message events."""
|
||||
updates = [
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_text(
|
||||
text='{"recipe_title": "Pasta", "ingredients": ["pasta"], "message": "Enjoy your pasta!"}'
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
# Text should be emitted from the message field
|
||||
text_contents = stream.get("TEXT_MESSAGE_CONTENT")
|
||||
if text_contents:
|
||||
combined = "".join(getattr(e, "delta", "") for e in text_contents)
|
||||
assert "Enjoy your pasta!" in combined
|
||||
@@ -0,0 +1,207 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Golden event-stream tests for the workflow HITL (subgraphs) scenario.
|
||||
|
||||
Extends the existing test_subgraphs_example_agent.py with EventStream assertions
|
||||
on full event ordering, balancing, and interrupt structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui_examples.agents.subgraphs_agent import subgraphs_agent
|
||||
|
||||
|
||||
async def _run(agent: Any, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in agent.run(payload)])
|
||||
|
||||
|
||||
# ── Turn 1: Initial request → flight interrupt ──
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_golden_bookends() -> None:
|
||||
"""Turn 1 starts with RUN_STARTED and ends with RUN_FINISHED."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-1",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip to San Francisco"}],
|
||||
},
|
||||
)
|
||||
stream.assert_bookends()
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_no_errors() -> None:
|
||||
"""Turn 1 completes without errors."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-2",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip"}],
|
||||
},
|
||||
)
|
||||
stream.assert_no_run_error()
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_has_step_events() -> None:
|
||||
"""Turn 1 emits STEP_STARTED and STEP_FINISHED for workflow executors."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-3",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip"}],
|
||||
},
|
||||
)
|
||||
stream.assert_has_type("STEP_STARTED")
|
||||
stream.assert_has_type("STEP_FINISHED")
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_interrupt_structure() -> None:
|
||||
"""Turn 1 RUN_FINISHED carries flight interrupt with correct structure."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-4",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip to SF"}],
|
||||
},
|
||||
)
|
||||
|
||||
interrupt = stream.run_finished_interrupts()
|
||||
assert len(interrupt) > 0
|
||||
interrupt_value = stream.interrupt_metadata_value(interrupt[0])
|
||||
assert interrupt_value["agent"] == "flights"
|
||||
assert len(interrupt_value["options"]) == 2
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_text_messages_balanced() -> None:
|
||||
"""All text messages in turn 1 are properly balanced."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-5",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip"}],
|
||||
},
|
||||
)
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
|
||||
async def test_subgraphs_turn1_ordered_flow() -> None:
|
||||
"""Turn 1 event ordering: RUN_STARTED → STATE_SNAPSHOT → STEP_* → TOOL_CALL_* → RUN_FINISHED."""
|
||||
agent = subgraphs_agent()
|
||||
stream = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": "thread-sub-golden-6",
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip"}],
|
||||
},
|
||||
)
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"STATE_SNAPSHOT",
|
||||
"STEP_STARTED",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ── Multi-turn: Flight selection → hotel interrupt → completion ──
|
||||
|
||||
|
||||
async def test_subgraphs_full_flow_event_ordering() -> None:
|
||||
"""Complete 3-turn flow maintains proper event ordering throughout."""
|
||||
agent = subgraphs_agent()
|
||||
thread_id = "thread-sub-golden-full"
|
||||
|
||||
# Turn 1
|
||||
stream1 = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"run_id": "run-1",
|
||||
"messages": [{"role": "user", "content": "Plan a trip to SF from Amsterdam"}],
|
||||
},
|
||||
)
|
||||
stream1.assert_bookends()
|
||||
stream1.assert_no_run_error()
|
||||
|
||||
# Extract flight interrupt
|
||||
interrupt1 = stream1.run_finished_interrupts()[0]
|
||||
|
||||
# Turn 2: Select flight
|
||||
stream2 = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"run_id": "run-2",
|
||||
"resume": {
|
||||
"interrupts": [
|
||||
{
|
||||
"id": interrupt1["id"],
|
||||
"value": json.dumps(
|
||||
{
|
||||
"airline": "United",
|
||||
"departure": "Amsterdam (AMS)",
|
||||
"arrival": "San Francisco (SFO)",
|
||||
"price": "$720",
|
||||
"duration": "12h 15m",
|
||||
}
|
||||
),
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
stream2.assert_bookends()
|
||||
stream2.assert_no_run_error()
|
||||
|
||||
# Should now have hotel interrupt
|
||||
interrupt2 = stream2.run_finished_interrupts()
|
||||
assert stream2.interrupt_metadata_value(interrupt2[0])["agent"] == "hotels"
|
||||
|
||||
# Turn 3: Select hotel
|
||||
stream3 = await _run(
|
||||
agent,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"run_id": "run-3",
|
||||
"resume": {
|
||||
"interrupts": [
|
||||
{
|
||||
"id": interrupt2[0]["id"],
|
||||
"value": json.dumps(
|
||||
{
|
||||
"name": "The Ritz-Carlton",
|
||||
"location": "Nob Hill",
|
||||
"price_per_night": "$550/night",
|
||||
"rating": "4.8 stars",
|
||||
}
|
||||
),
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
stream3.assert_bookends()
|
||||
stream3.assert_no_run_error()
|
||||
stream3.assert_text_messages_balanced()
|
||||
|
||||
# Final turn should not have interrupt
|
||||
finished3 = stream3.last("RUN_FINISHED")
|
||||
final_dump = finished3.model_dump(by_alias=True, exclude_none=True)
|
||||
assert "outcome" not in final_dump, f"Expected no interrupt after completion, got {final_dump.get('outcome')}"
|
||||
@@ -0,0 +1,945 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Comprehensive golden event-stream tests for AgentFrameworkWorkflow.
|
||||
|
||||
Covers the full matrix of workflow-specific AG-UI patterns:
|
||||
- request_info → TOOL_CALL lifecycle and balancing
|
||||
- Executor step events and activity snapshots
|
||||
- Text output, dict output, BaseEvent passthrough, AgentResponse output
|
||||
- Text deduplication across workflow outputs
|
||||
- Workflow error handling → RUN_ERROR
|
||||
- Multi-turn interrupt/resume round-trips
|
||||
- Empty turns with pending requests
|
||||
- Custom workflow events
|
||||
- Text message draining on request_info and executor boundaries
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, cast
|
||||
|
||||
from ag_ui.core import EventType, StateSnapshotEvent
|
||||
from agent_framework import (
|
||||
AgentResponse,
|
||||
Content,
|
||||
Executor,
|
||||
Message,
|
||||
WorkflowBuilder,
|
||||
WorkflowContext,
|
||||
WorkflowEvent,
|
||||
executor,
|
||||
handler,
|
||||
response_handler,
|
||||
)
|
||||
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
|
||||
|
||||
from agent_framework_ag_ui import AgentFrameworkWorkflow
|
||||
|
||||
|
||||
async def _run(wrapper: AgentFrameworkWorkflow, payload: dict[str, Any]) -> EventStream:
|
||||
return EventStream([event async for event in wrapper.run(payload)])
|
||||
|
||||
|
||||
def _payload(
|
||||
msg: str = "go",
|
||||
*,
|
||||
thread_id: str = "thread-wf",
|
||||
run_id: str = "run-wf",
|
||||
**extra: Any,
|
||||
) -> dict[str, Any]:
|
||||
return {"thread_id": thread_id, "run_id": run_id, "messages": [{"role": "user", "content": msg}], **extra}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 1. Basic workflow text output
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_text_output_golden_sequence() -> None:
|
||||
"""Simple text output: RUN_STARTED → STEP_STARTED → TEXT_* → STEP_FINISHED → TEXT_MESSAGE_END → RUN_FINISHED."""
|
||||
|
||||
@executor(id="greeter")
|
||||
async def greeter(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("Hello from workflow!")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=greeter).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_text_messages_balanced()
|
||||
stream.assert_has_type("TEXT_MESSAGE_START")
|
||||
stream.assert_has_type("TEXT_MESSAGE_CONTENT")
|
||||
stream.assert_has_type("TEXT_MESSAGE_END")
|
||||
|
||||
# Verify actual content
|
||||
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert "Hello from workflow!" in deltas
|
||||
|
||||
|
||||
async def test_workflow_text_output_message_id_consistency() -> None:
|
||||
"""All text events for a single output share the same message_id."""
|
||||
|
||||
@executor(id="echo")
|
||||
async def echo(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("echo reply")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=echo).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_message_ids_consistent()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 2. Executor step events and activity snapshots
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_executor_lifecycle_events() -> None:
|
||||
"""Executor invocation produces STEP_STARTED, ACTIVITY_SNAPSHOT, STEP_FINISHED."""
|
||||
|
||||
@executor(id="worker")
|
||||
async def worker(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("done")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=worker).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
# Step events with executor ID
|
||||
started = [e for e in stream.get("STEP_STARTED") if getattr(e, "step_name", "") == "worker"]
|
||||
finished = [e for e in stream.get("STEP_FINISHED") if getattr(e, "step_name", "") == "worker"]
|
||||
assert started, "Expected STEP_STARTED for 'worker'"
|
||||
assert finished, "Expected STEP_FINISHED for 'worker'"
|
||||
|
||||
# Activity snapshots
|
||||
activities = stream.get("ACTIVITY_SNAPSHOT")
|
||||
assert activities, "Expected ACTIVITY_SNAPSHOT events"
|
||||
# Check one of them has executor payload
|
||||
executor_activities = [a for a in activities if getattr(a, "activity_type", None) == "executor"]
|
||||
assert executor_activities, "Expected executor-type activity snapshots"
|
||||
|
||||
|
||||
async def test_workflow_executor_step_ordering() -> None:
|
||||
"""STEP_STARTED comes before content, STEP_FINISHED comes after."""
|
||||
|
||||
@executor(id="orderer")
|
||||
async def orderer(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("ordered output")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=orderer).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"STEP_STARTED",
|
||||
"TEXT_MESSAGE_START",
|
||||
"TEXT_MESSAGE_CONTENT",
|
||||
"STEP_FINISHED",
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 3. Dict output → CUSTOM workflow_output
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_dict_output_maps_to_custom_event() -> None:
|
||||
"""Non-chat dict output is emitted as CUSTOM workflow_output event."""
|
||||
|
||||
@executor(id="structured")
|
||||
async def structured(message: Any, ctx: WorkflowContext[Any, dict[str, int]]) -> None:
|
||||
await ctx.yield_output({"count": 42, "status": 1})
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=structured).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
customs = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "workflow_output"]
|
||||
assert len(customs) == 1
|
||||
assert customs[0].value == {"count": 42, "status": 1}
|
||||
|
||||
# Should NOT have TEXT_MESSAGE events for dict output
|
||||
assert "TEXT_MESSAGE_CONTENT" not in stream.types()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 4. BaseEvent passthrough
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_base_event_passthrough() -> None:
|
||||
"""AG-UI BaseEvent outputs are yielded directly, not wrapped."""
|
||||
|
||||
@executor(id="stateful")
|
||||
async def stateful(message: Any, ctx: WorkflowContext[Any, StateSnapshotEvent]) -> None:
|
||||
await ctx.yield_output(StateSnapshotEvent(type=EventType.STATE_SNAPSHOT, snapshot={"active_agent": "flights"}))
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=stateful).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
snapshots = stream.get("STATE_SNAPSHOT")
|
||||
assert len(snapshots) == 1
|
||||
assert snapshots[0].snapshot["active_agent"] == "flights"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 5. AgentResponse output (conversation payload)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_agent_response_output_extracts_latest_assistant() -> None:
|
||||
"""AgentResponse output uses only the latest assistant message, not full history."""
|
||||
|
||||
@executor(id="responder")
|
||||
async def responder(message: Any, ctx: WorkflowContext[Any, AgentResponse]) -> None:
|
||||
response = AgentResponse(
|
||||
messages=[
|
||||
Message(role="user", contents=[Content.from_text("My order is damaged")]),
|
||||
Message(role="assistant", contents=[Content.from_text("I'll process your replacement.")]),
|
||||
]
|
||||
)
|
||||
await ctx.yield_output(response)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=responder).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_text_messages_balanced()
|
||||
|
||||
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert deltas == ["I'll process your replacement."]
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 6. Custom workflow events
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ProgressEvent(WorkflowEvent):
|
||||
"""Custom workflow event for testing CUSTOM event mapping."""
|
||||
|
||||
def __init__(self, progress: int) -> None:
|
||||
super().__init__(cast(Any, "custom_progress"), data={"progress": progress})
|
||||
|
||||
|
||||
async def test_workflow_custom_events() -> None:
|
||||
"""Custom workflow events are mapped to CUSTOM AG-UI events."""
|
||||
|
||||
@executor(id="progress_tracker")
|
||||
async def progress_tracker(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.add_event(ProgressEvent(25))
|
||||
await ctx.yield_output("In progress...")
|
||||
await ctx.add_event(ProgressEvent(100))
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=progress_tracker).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
progress_events = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "custom_progress"]
|
||||
assert len(progress_events) == 2
|
||||
assert progress_events[0].value == {"progress": 25}
|
||||
assert progress_events[1].value == {"progress": 100}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 7. request_info → TOOL_CALL lifecycle
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_request_info_tool_call_lifecycle() -> None:
|
||||
"""request_info emits TOOL_CALL_START/ARGS/END cycle plus CUSTOM request_info."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info("Need approval", str, request_id="req-1")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
# Tool call lifecycle
|
||||
stream.assert_ordered_types(
|
||||
[
|
||||
"RUN_STARTED",
|
||||
"TOOL_CALL_START",
|
||||
"TOOL_CALL_ARGS",
|
||||
"TOOL_CALL_END",
|
||||
"CUSTOM", # request_info
|
||||
"RUN_FINISHED",
|
||||
]
|
||||
)
|
||||
|
||||
# Verify tool call details
|
||||
start = stream.first("TOOL_CALL_START")
|
||||
assert start.tool_call_id == "req-1"
|
||||
assert start.tool_call_name == "request_info"
|
||||
|
||||
# TOOL_CALL_ARGS should contain the request payload
|
||||
args = stream.first("TOOL_CALL_ARGS")
|
||||
assert args.tool_call_id == "req-1"
|
||||
parsed_args = json.loads(args.delta)
|
||||
assert parsed_args["request_id"] == "req-1"
|
||||
|
||||
# Tool calls should be balanced
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
|
||||
async def test_workflow_request_info_interrupt_in_run_finished() -> None:
|
||||
"""request_info populates RUN_FINISHED.outcome.interrupts with the request metadata."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info(
|
||||
{"message": "Choose a flight", "options": [{"airline": "KLM"}], "agent": "flights"},
|
||||
dict,
|
||||
request_id="flights-choice",
|
||||
)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
interrupt = stream.run_finished_interrupts()
|
||||
assert len(interrupt) == 1
|
||||
assert interrupt[0]["id"] == "flights-choice"
|
||||
assert stream.interrupt_metadata_value(interrupt[0])["agent"] == "flights"
|
||||
|
||||
|
||||
async def test_workflow_request_info_emits_interrupt_card_event() -> None:
|
||||
"""request_info with dict data emits a WorkflowInterruptEvent custom event."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info(
|
||||
{"message": "Pick one", "options": ["A", "B"]},
|
||||
dict,
|
||||
request_id="pick-1",
|
||||
)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
interrupt_cards = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "WorkflowInterruptEvent"]
|
||||
assert interrupt_cards, "Expected WorkflowInterruptEvent custom event"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 8. Text message draining on request_info boundary
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_text_drained_before_request_info() -> None:
|
||||
"""Open text message is closed (TEXT_MESSAGE_END) before request_info tool calls begin."""
|
||||
|
||||
@executor(id="text_then_request")
|
||||
async def text_then_request(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.yield_output("Please confirm this action.") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
|
||||
await ctx.request_info("Need approval", str, request_id="approval-1")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=text_then_request).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
# TEXT_MESSAGE_END must appear before TOOL_CALL_START
|
||||
types = stream.types()
|
||||
text_end_idx = types.index("TEXT_MESSAGE_END")
|
||||
tool_start_idx = types.index("TOOL_CALL_START")
|
||||
assert text_end_idx < tool_start_idx, (
|
||||
f"TEXT_MESSAGE_END (idx={text_end_idx}) must come before TOOL_CALL_START (idx={tool_start_idx})"
|
||||
)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 9. Text deduplication
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_skips_duplicate_text_from_snapshot() -> None:
|
||||
"""Duplicate text from AgentResponse snapshot is not re-emitted."""
|
||||
|
||||
@executor(id="deduper")
|
||||
async def deduper(message: Any, ctx: WorkflowContext[Any, Any]) -> None:
|
||||
text = "Order processed successfully."
|
||||
await ctx.yield_output(text)
|
||||
# Snapshot repeats the same text
|
||||
await ctx.yield_output(
|
||||
AgentResponse(
|
||||
messages=[
|
||||
Message(role="user", contents=[Content.from_text("process order")]),
|
||||
Message(role="assistant", contents=[Content.from_text(text)]),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=deduper).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
|
||||
# Text should appear only once
|
||||
assert deltas == ["Order processed successfully."]
|
||||
|
||||
|
||||
async def test_workflow_skips_consecutive_duplicate_outputs() -> None:
|
||||
"""Consecutive identical text outputs are deduplicated."""
|
||||
|
||||
@executor(id="repeater")
|
||||
async def repeater(message: Any, ctx: WorkflowContext[Any, Any]) -> None:
|
||||
text = "Done!"
|
||||
await ctx.yield_output(text)
|
||||
await ctx.yield_output(text)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=repeater).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert deltas == ["Done!"]
|
||||
|
||||
|
||||
async def test_workflow_emits_distinct_consecutive_outputs() -> None:
|
||||
"""Distinct text outputs are all emitted, not incorrectly deduplicated."""
|
||||
|
||||
@executor(id="multisayer")
|
||||
async def multisayer(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("First part. ")
|
||||
await ctx.yield_output("Second part.")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=multisayer).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_text_messages_balanced()
|
||||
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert deltas == ["First part. ", "Second part."]
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 10. Workflow error handling → RUN_ERROR
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_error_emits_run_error_event() -> None:
|
||||
"""Exceptions during workflow streaming produce RUN_ERROR events."""
|
||||
|
||||
class FailingWorkflow:
|
||||
def run(self, **kwargs: Any):
|
||||
async def _stream():
|
||||
raise RuntimeError("workflow exploded")
|
||||
yield # pragma: no cover
|
||||
|
||||
return _stream()
|
||||
|
||||
wrapper = AgentFrameworkWorkflow(workflow=cast(Any, FailingWorkflow()))
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
# Should still have RUN_STARTED
|
||||
stream.assert_has_type("RUN_STARTED")
|
||||
# Should have RUN_ERROR
|
||||
stream.assert_has_type("RUN_ERROR")
|
||||
error = stream.first("RUN_ERROR")
|
||||
assert "workflow exploded" in error.message
|
||||
|
||||
|
||||
async def test_workflow_error_preserves_bookend_structure() -> None:
|
||||
"""Even on error, RUN_STARTED is the first event."""
|
||||
|
||||
class FailingWorkflow:
|
||||
def run(self, **kwargs: Any):
|
||||
async def _stream():
|
||||
raise ValueError("bad input")
|
||||
yield # pragma: no cover
|
||||
|
||||
return _stream()
|
||||
|
||||
wrapper = AgentFrameworkWorkflow(workflow=cast(Any, FailingWorkflow()))
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
types = stream.types()
|
||||
assert types[0] == "RUN_STARTED"
|
||||
assert "RUN_ERROR" in types
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 11. Multi-turn request_info interrupt/resume
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_interrupt_resume_round_trip() -> None:
|
||||
"""Turn 1: request_info → interrupt. Turn 2: resume → completion."""
|
||||
|
||||
class RequesterExecutor(Executor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="requester")
|
||||
|
||||
@handler
|
||||
async def start(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info("Choose an option", str, request_id="choice-1")
|
||||
|
||||
@response_handler
|
||||
async def handle_choice(self, original: str, response: str, ctx: WorkflowContext) -> None:
|
||||
await ctx.yield_output(f"You chose: {response}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=RequesterExecutor()).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1
|
||||
stream1 = await _run(wrapper, _payload(thread_id="thread-resume", run_id="run-1"))
|
||||
stream1.assert_bookends()
|
||||
stream1.assert_no_run_error()
|
||||
stream1.assert_tool_calls_balanced()
|
||||
|
||||
interrupt1 = stream1.run_finished_interrupts()
|
||||
assert interrupt1, "Expected interrupt"
|
||||
assert interrupt1[0]["id"] == "choice-1"
|
||||
|
||||
# Turn 2: resume
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-resume",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
"resume": {"interrupts": [{"id": "choice-1", "value": "Option A"}]},
|
||||
},
|
||||
)
|
||||
stream2.assert_has_run_lifecycle()
|
||||
stream2.assert_no_run_error()
|
||||
stream2.assert_text_messages_balanced()
|
||||
|
||||
# Should have the response text
|
||||
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert any("Option A" in d for d in deltas), f"Expected 'Option A' in deltas: {deltas}"
|
||||
|
||||
# No interrupt after resume
|
||||
finished2 = stream2.last("RUN_FINISHED")
|
||||
dumped2 = finished2.model_dump(by_alias=True, exclude_none=True)
|
||||
assert "outcome" not in dumped2
|
||||
|
||||
|
||||
async def test_workflow_forwarded_props_resume() -> None:
|
||||
"""forwarded_props.command.resume should resume with canonical resume entries."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info({"options": [{"name": "A"}]}, dict, request_id="pick")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1
|
||||
await _run(wrapper, _payload(thread_id="thread-fwd", run_id="run-1"))
|
||||
|
||||
# Turn 2 via forwarded_props
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-fwd",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
"forwarded_props": {
|
||||
"command": {"resume": [{"interruptId": "pick", "status": "resolved", "payload": {"name": "A"}}]}
|
||||
},
|
||||
},
|
||||
)
|
||||
stream2.assert_bookends()
|
||||
stream2.assert_no_run_error()
|
||||
|
||||
finished = stream2.last("RUN_FINISHED")
|
||||
assert "outcome" not in finished.model_dump(by_alias=True, exclude_none=True)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 12. Empty turns with pending requests
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_empty_turn_with_pending_request_emits_run_error() -> None:
|
||||
"""An empty turn with a pending request must provide canonical resume entries."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info({"prompt": "choose"}, dict, request_id="pick-one")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1: trigger the request
|
||||
await _run(wrapper, _payload(thread_id="thread-empty", run_id="run-1"))
|
||||
|
||||
# Turn 2: empty messages, no resume
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-empty",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
},
|
||||
)
|
||||
stream2.assert_has_type("RUN_STARTED")
|
||||
run_error = stream2.last("RUN_ERROR")
|
||||
assert run_error.code == "WORKFLOW_RESUME_REQUIRED"
|
||||
|
||||
|
||||
async def test_workflow_empty_turn_no_pending_requests() -> None:
|
||||
"""Empty turn with no pending requests produces clean bookends."""
|
||||
|
||||
@executor(id="noop")
|
||||
async def noop(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output("done")
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=noop).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Run once to completion
|
||||
await _run(wrapper, _payload(thread_id="thread-empty-clean", run_id="run-1"))
|
||||
|
||||
# Empty turn
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-empty-clean",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
},
|
||||
)
|
||||
stream2.assert_bookends()
|
||||
stream2.assert_no_run_error()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 13. Usage content as CUSTOM event
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_usage_output_maps_to_custom_event() -> None:
|
||||
"""Usage Content outputs are surfaced as custom usage events."""
|
||||
|
||||
@executor(id="usage_reporter")
|
||||
async def usage_reporter(message: Any, ctx: WorkflowContext[Any, Content]) -> None:
|
||||
await ctx.yield_output(
|
||||
Content.from_usage({"input_token_count": 100, "output_token_count": 50, "total_token_count": 150})
|
||||
)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=usage_reporter).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
stream = await _run(wrapper, _payload())
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
|
||||
usage_events = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "usage"]
|
||||
assert len(usage_events) == 1
|
||||
assert usage_events[0].value["input_token_count"] == 100
|
||||
assert usage_events[0].value["total_token_count"] == 150
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 14. Approval flow (Content-based request_info)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_approval_flow_round_trip() -> None:
|
||||
"""function_approval_request via request_info, then resume with approval response."""
|
||||
|
||||
class ApprovalExecutor(Executor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="approval_exec")
|
||||
|
||||
@handler
|
||||
async def start(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
function_call = Content.from_function_call(
|
||||
call_id="refund-call",
|
||||
name="submit_refund",
|
||||
arguments={"order_id": "12345", "amount": "$89.99"},
|
||||
)
|
||||
approval_request = Content.from_function_approval_request(id="approval-1", function_call=function_call)
|
||||
await ctx.request_info(approval_request, Content, request_id="approval-1")
|
||||
|
||||
@response_handler
|
||||
async def handle_approval(self, original_request: Content, response: Content, ctx: WorkflowContext) -> None:
|
||||
status = "approved" if bool(response.approved) else "rejected"
|
||||
await ctx.yield_output(f"Refund {status}.") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=ApprovalExecutor()).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1: request approval
|
||||
stream1 = await _run(wrapper, _payload(thread_id="thread-approval", run_id="run-1"))
|
||||
stream1.assert_bookends()
|
||||
stream1.assert_no_run_error()
|
||||
|
||||
interrupt1 = stream1.run_finished_interrupts()
|
||||
assert interrupt1, "Expected approval interrupt"
|
||||
interrupt_value = stream1.interrupt_metadata_value(interrupt1[0])
|
||||
|
||||
# Turn 2: approve
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-approval",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
"resume": {
|
||||
"interrupts": [
|
||||
{
|
||||
"id": "approval-1",
|
||||
"value": {
|
||||
"type": "function_approval_response",
|
||||
"approved": True,
|
||||
"id": interrupt_value.get("id", "approval-1"),
|
||||
"function_call": interrupt_value.get("function_call"),
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
stream2.assert_has_run_lifecycle()
|
||||
stream2.assert_no_run_error()
|
||||
stream2.assert_text_messages_balanced()
|
||||
|
||||
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert any("approved" in d for d in deltas)
|
||||
|
||||
# No more interrupt
|
||||
finished2 = stream2.last("RUN_FINISHED")
|
||||
assert "outcome" not in finished2.model_dump(by_alias=True, exclude_none=True)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 15. Message list request/response coercion
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_message_list_resume() -> None:
|
||||
"""Resume with list[Message] payload coerces correctly into workflow response."""
|
||||
|
||||
class MessageRequestExecutor(Executor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="msg_request")
|
||||
|
||||
@handler
|
||||
async def start(self, message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info({"prompt": "Need follow-up"}, list[Message], request_id="handoff")
|
||||
|
||||
@response_handler
|
||||
async def handle_input(self, original: dict, response: list[Message], ctx: WorkflowContext) -> None:
|
||||
user_text = response[0].text if response else ""
|
||||
await ctx.yield_output(f"Got: {user_text}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=MessageRequestExecutor()).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1
|
||||
await _run(wrapper, _payload(thread_id="thread-msg", run_id="run-1"))
|
||||
|
||||
# Turn 2: resume with message list
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-msg",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
"resume": {
|
||||
"interrupts": [
|
||||
{
|
||||
"id": "handoff",
|
||||
"value": [
|
||||
{"role": "user", "contents": [{"type": "text", "text": "Ship a replacement"}]},
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
stream2.assert_has_run_lifecycle()
|
||||
stream2.assert_no_run_error()
|
||||
stream2.assert_text_messages_balanced()
|
||||
|
||||
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert any("replacement" in d for d in deltas)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 16. Plain text follow-up does NOT infer interrupt response
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_plain_text_does_not_resume_pending_dict_request() -> None:
|
||||
"""Plain text user follow-up should fail instead of being coerced into a dict response."""
|
||||
|
||||
@executor(id="requester")
|
||||
async def requester(message: Any, ctx: WorkflowContext) -> None:
|
||||
await ctx.request_info(
|
||||
{"message": "Choose a flight", "options": [{"airline": "KLM"}], "agent": "flights"},
|
||||
dict,
|
||||
request_id="flights-choice",
|
||||
)
|
||||
|
||||
workflow = WorkflowBuilder(start_executor=requester).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1
|
||||
await _run(wrapper, _payload(thread_id="thread-nocoerce", run_id="run-1"))
|
||||
|
||||
# Turn 2: plain text follow-up with request_info tool call in history
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-nocoerce",
|
||||
"run_id": "run-2",
|
||||
"messages": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "flights-choice",
|
||||
"type": "function",
|
||||
"function": {"name": "request_info", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "I prefer KLM please"},
|
||||
],
|
||||
},
|
||||
)
|
||||
stream2.assert_has_type("RUN_STARTED")
|
||||
run_error = stream2.last("RUN_ERROR")
|
||||
assert run_error.code == "WORKFLOW_RESUME_REQUIRED"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 17. Workflow factory (thread-scoped workflows)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_factory_thread_scoping() -> None:
|
||||
"""workflow_factory creates separate workflow instances per thread_id."""
|
||||
|
||||
def make_workflow(thread_id: str):
|
||||
@executor(id="echo")
|
||||
async def echo(message: Any, ctx: WorkflowContext[Any, str]) -> None:
|
||||
await ctx.yield_output(f"Thread: {thread_id}")
|
||||
|
||||
return WorkflowBuilder(start_executor=echo).build()
|
||||
|
||||
wrapper = AgentFrameworkWorkflow(workflow_factory=make_workflow)
|
||||
|
||||
stream_a = await _run(wrapper, _payload(thread_id="thread-a", run_id="run-a"))
|
||||
stream_b = await _run(wrapper, _payload(thread_id="thread-b", run_id="run-b"))
|
||||
|
||||
stream_a.assert_bookends()
|
||||
stream_b.assert_bookends()
|
||||
|
||||
deltas_a = [e.delta for e in stream_a.get("TEXT_MESSAGE_CONTENT")]
|
||||
deltas_b = [e.delta for e in stream_b.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert any("thread-a" in d for d in deltas_a)
|
||||
assert any("thread-b" in d for d in deltas_b)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 18. Multiple request_info calls in sequence
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_workflow_sequential_request_info_interrupts() -> None:
|
||||
"""Two chained executors each requesting info: first triggers interrupt, resume, then second triggers interrupt.
|
||||
|
||||
This mirrors the subgraphs_agent pattern where separate executors handle sequential interactions.
|
||||
"""
|
||||
|
||||
class NameRequester(Executor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="name_requester")
|
||||
|
||||
@handler
|
||||
async def start(self, message: Any, ctx: WorkflowContext[str]) -> None:
|
||||
await ctx.request_info("What's your name?", str, request_id="name-req")
|
||||
|
||||
@response_handler
|
||||
async def handle_name(self, original: str, response: str, ctx: WorkflowContext[str]) -> None:
|
||||
await ctx.send_message(response)
|
||||
|
||||
class DestRequester(Executor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(id="dest_requester")
|
||||
|
||||
@handler
|
||||
async def start(self, message: str, ctx: WorkflowContext[str]) -> None:
|
||||
self._name = message
|
||||
await ctx.request_info("Where to?", str, request_id="dest-req")
|
||||
|
||||
@response_handler
|
||||
async def handle_dest(self, original: str, response: str, ctx: WorkflowContext[str]) -> None:
|
||||
await ctx.yield_output(f"Booking for {self._name} to {response}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
|
||||
|
||||
name_requester = NameRequester()
|
||||
dest_requester = DestRequester()
|
||||
workflow = WorkflowBuilder(start_executor=name_requester).add_chain([name_requester, dest_requester]).build()
|
||||
wrapper = AgentFrameworkWorkflow(workflow=workflow)
|
||||
|
||||
# Turn 1
|
||||
stream1 = await _run(wrapper, _payload(thread_id="thread-seq", run_id="run-1"))
|
||||
stream1.assert_bookends()
|
||||
stream1.assert_tool_calls_balanced()
|
||||
interrupt1 = stream1.run_finished_interrupts()
|
||||
assert interrupt1[0]["id"] == "name-req"
|
||||
|
||||
# Turn 2: answer name → triggers second executor's request_info
|
||||
stream2 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-seq",
|
||||
"run_id": "run-2",
|
||||
"messages": [],
|
||||
"resume": {"interrupts": [{"id": "name-req", "value": "Alice"}]},
|
||||
},
|
||||
)
|
||||
stream2.assert_has_run_lifecycle()
|
||||
stream2.assert_tool_calls_balanced()
|
||||
interrupt2 = stream2.run_finished_interrupts()
|
||||
assert interrupt2[0]["id"] == "dest-req"
|
||||
|
||||
# Turn 3: answer destination → completion
|
||||
stream3 = await _run(
|
||||
wrapper,
|
||||
{
|
||||
"thread_id": "thread-seq",
|
||||
"run_id": "run-3",
|
||||
"messages": [],
|
||||
"resume": {"interrupts": [{"id": "dest-req", "value": "Paris"}]},
|
||||
},
|
||||
)
|
||||
stream3.assert_has_run_lifecycle()
|
||||
stream3.assert_no_run_error()
|
||||
stream3.assert_text_messages_balanced()
|
||||
|
||||
deltas = [e.delta for e in stream3.get("TEXT_MESSAGE_CONTENT")]
|
||||
assert any("Alice" in d and "Paris" in d for d in deltas)
|
||||
assert "outcome" not in stream3.last("RUN_FINISHED").model_dump(by_alias=True, exclude_none=True)
|
||||
Reference in New Issue
Block a user