chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:25 +08:00
commit db620d33df
5151 changed files with 925932 additions and 0 deletions
@@ -0,0 +1 @@
# Copyright (c) Microsoft. All rights reserved.
@@ -0,0 +1,13 @@
# Copyright (c) Microsoft. All rights reserved.
"""Conftest for golden tests — ensures parent test dir is importable."""
import sys
from pathlib import Path
def pytest_configure() -> None:
"""Ensure parent test directory is on sys.path for helper module imports."""
parent_test_dir = str(Path(__file__).resolve().parent.parent)
if parent_test_dir not in sys.path:
sys.path.insert(0, parent_test_dir)
@@ -0,0 +1,140 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the basic agentic chat scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
return AgentFrameworkAgent(agent=stub, **kwargs)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
BASIC_PAYLOAD: dict[str, Any] = {
"thread_id": "thread-chat",
"run_id": "run-chat",
"messages": [{"role": "user", "content": "Hello"}],
}
def _text_update(text: str) -> AgentResponseUpdate:
return AgentResponseUpdate(contents=[Content.from_text(text=text)], role="assistant")
def _snapshot_role(msg: Any) -> str:
"""Extract role string from a snapshot message (Pydantic model or dict)."""
role = getattr(msg, "role", None) or (msg.get("role") if isinstance(msg, dict) else None)
if role is None:
return ""
return str(getattr(role, "value", role))
def _snapshot_content(msg: Any) -> str:
"""Extract content string from a snapshot message."""
content = getattr(msg, "content", None) or (msg.get("content") if isinstance(msg, dict) else "")
return str(content) if content else ""
# ── Golden stream tests ──
async def test_basic_chat_golden_event_sequence() -> None:
"""Assert the exact event type sequence for a single text response."""
agent = _build_agent([_text_update("Hi there!")])
stream = await _run(agent, BASIC_PAYLOAD)
stream.assert_strict_types(
[
"RUN_STARTED",
"TEXT_MESSAGE_START",
"TEXT_MESSAGE_CONTENT",
"TEXT_MESSAGE_END",
"MESSAGES_SNAPSHOT",
"RUN_FINISHED",
]
)
async def test_basic_chat_bookends() -> None:
"""RUN_STARTED is first, RUN_FINISHED is last."""
agent = _build_agent([_text_update("reply")])
stream = await _run(agent, BASIC_PAYLOAD)
stream.assert_bookends()
async def test_basic_chat_text_messages_balanced() -> None:
"""Every TEXT_MESSAGE_START has a matching TEXT_MESSAGE_END."""
agent = _build_agent([_text_update("reply")])
stream = await _run(agent, BASIC_PAYLOAD)
stream.assert_text_messages_balanced()
async def test_basic_chat_no_errors() -> None:
"""No RUN_ERROR events in a normal flow."""
agent = _build_agent([_text_update("reply")])
stream = await _run(agent, BASIC_PAYLOAD)
stream.assert_no_run_error()
async def test_basic_chat_message_id_consistency() -> None:
"""All text events reference the same message_id."""
agent = _build_agent([_text_update("reply")])
stream = await _run(agent, BASIC_PAYLOAD)
start = stream.first("TEXT_MESSAGE_START")
content = stream.first("TEXT_MESSAGE_CONTENT")
end = stream.first("TEXT_MESSAGE_END")
assert start.message_id == content.message_id == end.message_id
async def test_multi_chunk_text_golden_sequence() -> None:
"""Streaming multiple chunks produces START + multiple CONTENT + END."""
agent = _build_agent([_text_update("Hello "), _text_update("world!")])
stream = await _run(agent, BASIC_PAYLOAD)
stream.assert_strict_types(
[
"RUN_STARTED",
"TEXT_MESSAGE_START",
"TEXT_MESSAGE_CONTENT",
"TEXT_MESSAGE_CONTENT",
"TEXT_MESSAGE_END",
"MESSAGES_SNAPSHOT",
"RUN_FINISHED",
]
)
stream.assert_text_messages_balanced()
stream.assert_message_ids_consistent()
async def test_messages_snapshot_contains_assistant_reply() -> None:
"""MessagesSnapshotEvent includes the assistant's accumulated text."""
agent = _build_agent([_text_update("Hello there")])
stream = await _run(agent, BASIC_PAYLOAD)
snapshot = stream.messages_snapshot()
assistant_msgs = [m for m in snapshot if _snapshot_role(m) == "assistant"]
assert assistant_msgs, "No assistant message in snapshot"
assert any("Hello there" in _snapshot_content(m) for m in assistant_msgs)
async def test_empty_messages_produces_start_and_finish() -> None:
"""Empty message list still produces RUN_STARTED and RUN_FINISHED."""
agent = _build_agent([_text_update("reply")])
payload = {"thread_id": "t1", "run_id": "r1", "messages": []}
stream = await _run(agent, payload)
stream.assert_bookends()
assert "TEXT_MESSAGE_START" not in stream.types()
@@ -0,0 +1,236 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the backend (server-side) tools scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
return AgentFrameworkAgent(agent=stub, **kwargs)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-tools",
"run_id": "run-tools",
"messages": [{"role": "user", "content": "What's the weather?"}],
}
# ── Golden stream tests ──
async def test_tool_call_lifecycle_golden_sequence() -> None:
"""Assert the full event sequence for a tool call → result → text response."""
updates = [
# LLM calls the tool
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
role="assistant",
),
# Tool result comes back
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F and sunny")],
role="assistant",
),
# LLM responds with text
AgentResponseUpdate(
contents=[Content.from_text(text="It's 72°F and sunny in SF!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_ordered_types(
[
"RUN_STARTED",
"TEXT_MESSAGE_START", # Synthetic start for tool-only message
"TOOL_CALL_START",
"TOOL_CALL_ARGS",
"TOOL_CALL_END",
"TOOL_CALL_RESULT",
"TEXT_MESSAGE_END", # End of synthetic message
"TEXT_MESSAGE_START", # New message for text response
"TEXT_MESSAGE_CONTENT",
"TEXT_MESSAGE_END",
"MESSAGES_SNAPSHOT",
"RUN_FINISHED",
]
)
async def test_tool_calls_balanced() -> None:
"""Every TOOL_CALL_START has a matching TOOL_CALL_END."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_text(text="It's 72°F!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_tool_calls_balanced()
async def test_text_messages_balanced_with_tools() -> None:
"""Text messages are properly balanced even around tool calls."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city": "SF"}')],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_text(text="It's 72°F!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_text_messages_balanced()
async def test_tool_call_id_matches_result() -> None:
"""TOOL_CALL_START and TOOL_CALL_RESULT reference the same tool_call_id."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
start = stream.first("TOOL_CALL_START")
result = stream.first("TOOL_CALL_RESULT")
assert start.tool_call_id == result.tool_call_id == "call-1"
async def test_tool_result_content_preserved() -> None:
"""TOOL_CALL_RESULT event carries the tool's result content."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F and sunny")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
result = stream.first("TOOL_CALL_RESULT")
assert result.content == "72°F and sunny"
async def test_no_run_error_on_tool_flow() -> None:
"""Tool call flow doesn't produce RUN_ERROR."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments="{}")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_no_run_error()
stream.assert_bookends()
async def test_multiple_sequential_tool_calls() -> None:
"""Multiple sequential tool calls each produce balanced START/END pairs."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="tool_a", call_id="call-a", arguments="{}")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-a", result="result-a")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_call(name="tool_b", call_id="call-b", arguments="{}")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-b", result="result-b")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_text(text="Done!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_tool_calls_balanced()
stream.assert_text_messages_balanced()
stream.assert_bookends()
# Both tool calls should appear
starts = stream.get("TOOL_CALL_START")
assert len(starts) == 2
assert {s.tool_call_name for s in starts} == {"tool_a", "tool_b"}
async def test_messages_snapshot_includes_tool_calls() -> None:
"""MessagesSnapshotEvent includes tool call and result messages."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="get_weather", call_id="call-1", arguments='{"city":"SF"}')],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="72°F")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_text(text="It's warm!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_has_type("MESSAGES_SNAPSHOT")
snapshot = stream.messages_snapshot()
# Should have: user message, assistant with tool_calls, tool result, assistant text
assert len(snapshot) >= 3
@@ -0,0 +1,364 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the deterministic tool-driven state scenario.
Covers issue https://github.com/microsoft/agent-framework/issues/3167 — a tool
returning :func:`agent_framework_ag_ui.state_update` must push a deterministic
``StateSnapshotEvent`` derived from its actual return value, orthogonal to the
optimistic ``predict_state_config`` path. These golden tests pin the user-visible
event stream so additive changes cannot silently regress it.
"""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent, state_update
STATE_SCHEMA = {
"weather": {"type": "object", "description": "Last fetched weather"},
}
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
kwargs.setdefault("state_schema", STATE_SCHEMA)
return AgentFrameworkAgent(agent=stub, **kwargs)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-det-state",
"run_id": "run-det-state",
"messages": [{"role": "user", "content": "What's the weather in SF?"}],
"state": {"weather": {}},
}
def _tool_call(call_id: str, name: str, arguments: str) -> AgentResponseUpdate:
return AgentResponseUpdate(
contents=[Content.from_function_call(name=name, call_id=call_id, arguments=arguments)],
role="assistant",
)
def _tool_result_with_state(call_id: str, text: str, state: dict[str, Any]) -> AgentResponseUpdate:
"""Build a function_result update whose inner item carries a state marker.
This mirrors what the core framework produces when a real ``@tool`` returns
:func:`state_update`: ``parse_result`` keeps the ``Content`` as-is, and
``Content.from_function_result`` preserves its ``additional_properties``
inside ``items``.
"""
return AgentResponseUpdate(
contents=[
Content.from_function_result(
call_id=call_id,
result=[state_update(text=text, state=state)],
)
],
role="assistant",
)
def _tool_result_with_display(call_id: str, text: str, tool_result: Any, **kwargs: Any) -> AgentResponseUpdate:
"""Build a function_result update carrying an optional UI display marker."""
return AgentResponseUpdate(
contents=[
Content.from_function_result(
call_id=call_id,
result=[state_update(text=text, tool_result=tool_result, **kwargs)],
)
],
role="assistant",
)
# ── Golden stream tests ──
async def test_deterministic_state_emits_snapshot_after_tool_result() -> None:
"""The happy path: STATE_SNAPSHOT follows TOOL_CALL_RESULT in order."""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_state(
"call-1",
text="Weather in SF: 14°C foggy",
state={"weather": {"city": "SF", "temp": 14, "conditions": "foggy"}},
),
AgentResponseUpdate(
contents=[Content.from_text(text="It's 14°C and foggy in SF.")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_tool_calls_balanced()
stream.assert_text_messages_balanced()
# Ordered subsequence: the deterministic STATE_SNAPSHOT must follow the
# TOOL_CALL_RESULT. This is the central contract for #3167.
stream.assert_ordered_types(
[
"RUN_STARTED",
"TOOL_CALL_START",
"TOOL_CALL_ARGS",
"TOOL_CALL_END",
"TOOL_CALL_RESULT",
"STATE_SNAPSHOT",
"RUN_FINISHED",
]
)
# The final STATE_SNAPSHOT must carry the tool-driven state.
snapshot = stream.snapshot()
assert snapshot["weather"] == {"city": "SF", "temp": 14, "conditions": "foggy"}
async def test_deterministic_state_does_not_fire_for_plain_tool_result() -> None:
"""Regression guard: tools returning plain strings must NOT emit a new STATE_SNAPSHOT.
The initial STATE_SNAPSHOT fires once from the schema + initial payload
state. A plain (non-state_update) tool result must not add another one.
"""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
AgentResponseUpdate(
contents=[Content.from_function_result(call_id="call-1", result="14°C foggy")],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_text(text="It's 14°C and foggy.")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
snapshots = stream.get("STATE_SNAPSHOT")
# Only the initial snapshot (from state_schema + payload state) should exist.
# No deterministic snapshot should have been added by the plain tool result.
assert len(snapshots) == 1, (
f"Expected exactly 1 STATE_SNAPSHOT (initial only) for plain tool result; "
f"got {len(snapshots)}. Snapshots: {[s.snapshot for s in snapshots]}"
)
async def test_deterministic_state_merges_into_initial_state() -> None:
"""The tool-driven snapshot must merge into, not replace, pre-existing state keys."""
payload = dict(PAYLOAD)
payload["state"] = {"weather": {}, "user_preferences": {"unit": "C"}}
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_state(
"call-1",
text="Weather: 14°C",
state={"weather": {"city": "SF", "temp": 14}},
),
]
agent = _build_agent(updates, state_schema={**STATE_SCHEMA, "user_preferences": {"type": "object"}})
stream = await _run(agent, payload)
stream.assert_bookends()
stream.assert_no_run_error()
final_snapshot = stream.snapshot()
assert final_snapshot["weather"] == {"city": "SF", "temp": 14}
assert final_snapshot["user_preferences"] == {"unit": "C"}, (
"Pre-existing state keys must survive the deterministic merge"
)
async def test_deterministic_state_llm_visible_text_is_clean() -> None:
"""The LLM-visible TOOL_CALL_RESULT content must not leak the state marker key."""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_state(
"call-1",
text="Weather in SF: 14°C foggy",
state={"weather": {"city": "SF", "temp": 14}},
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
result = stream.first("TOOL_CALL_RESULT")
assert result.content == "Weather in SF: 14°C foggy"
# The marker key must never appear in the content sent back to the LLM.
assert "__ag_ui_tool_result_state__" not in result.content
assert "weather" not in result.content # not as a raw state dump
async def test_deterministic_state_multiple_tools_merge_in_order() -> None:
"""Two state-updating tools in one run merge in order; later wins on key collisions."""
updates = [
_tool_call("call-a", "get_weather", '{"city": "SF"}'),
_tool_result_with_state(
"call-a",
text="First result",
state={"weather": {"city": "SF", "temp": 14}, "source": "primary"},
),
_tool_call("call-b", "get_weather_refined", '{"city": "SF"}'),
_tool_result_with_state(
"call-b",
text="Refined result",
state={"source": "refined"},
),
AgentResponseUpdate(
contents=[Content.from_text(text="Here you go.")],
role="assistant",
),
]
agent = _build_agent(
updates,
state_schema={**STATE_SCHEMA, "source": {"type": "string"}},
)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_tool_calls_balanced()
stream.assert_no_run_error()
# Two tool-driven snapshots emitted (one per tool) plus the initial snapshot.
snapshots = stream.get("STATE_SNAPSHOT")
assert len(snapshots) >= 2, f"Expected at least 2 STATE_SNAPSHOTs; got {len(snapshots)}"
final = stream.snapshot()
assert final["weather"] == {"city": "SF", "temp": 14}
# Later tool must override earlier tool on the shared key.
assert final["source"] == "refined"
async def test_deterministic_state_coexists_with_predict_state_config() -> None:
"""Predictive state and deterministic state must coexist without clobbering each other."""
predict_config = {
"draft": {
"tool": "write_draft",
"tool_argument": "body",
}
}
updates = [
# Predictive tool: its argument "body" populates state.draft optimistically.
_tool_call("call-1", "write_draft", '{"body": "Hello world"}'),
# Then a deterministic tool result landing a different key.
_tool_result_with_state(
"call-1",
text="Draft saved",
state={"weather": {"city": "SF", "temp": 14}},
),
]
agent = _build_agent(
updates,
state_schema={**STATE_SCHEMA, "draft": {"type": "string"}},
predict_state_config=predict_config,
require_confirmation=False,
)
payload = dict(PAYLOAD)
payload["state"] = {"weather": {}, "draft": ""}
stream = await _run(agent, payload)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_tool_calls_balanced()
# The final observed state must contain both the deterministic and predictive contributions.
final = stream.snapshot()
assert final["weather"] == {"city": "SF", "temp": 14}, f"Deterministic state missing from final snapshot: {final}"
async def test_tool_result_display_payload_reaches_ui_event_only() -> None:
"""Rich display payload overrides TOOL_CALL_RESULT without leaking marker keys."""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_display(
"call-1",
text="Weather in SF: 14°C foggy",
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_tool_calls_balanced()
result = stream.first("TOOL_CALL_RESULT")
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
assert "__ag_ui_tool_result_display__" not in result.content
assert "__ag_ui_tool_result_state__" not in result.content
async def test_tool_result_display_falls_back_to_text_when_unset() -> None:
"""Without a display marker, the UI event keeps the existing text content."""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_state(
"call-1",
text="Weather in SF: 14°C foggy",
state={"weather": {"city": "SF", "temp": 14}},
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_tool_calls_balanced()
result = stream.first("TOOL_CALL_RESULT")
assert result.content == "Weather in SF: 14°C foggy"
assert "__ag_ui_tool_result_display__" not in result.content
assert "__ag_ui_tool_result_state__" not in result.content
async def test_tool_result_display_coexists_with_state_snapshot() -> None:
"""Display and durable state markers produce one deterministic state snapshot."""
updates = [
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
_tool_result_with_display(
"call-1",
text="Weather in SF: 14°C foggy",
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
state={"weather": {"city": "SF", "temp": 14, "conditions": "foggy"}},
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_tool_calls_balanced()
stream.assert_ordered_types(["TOOL_CALL_RESULT", "STATE_SNAPSHOT", "RUN_FINISHED"])
result = stream.first("TOOL_CALL_RESULT")
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
result_idx = stream.events.index(result)
deterministic_snapshots = [
event
for event in stream.events[result_idx + 1 :]
if getattr(getattr(event, "type", None), "value", getattr(event, "type", None)) == "STATE_SNAPSHOT"
]
assert len(deterministic_snapshots) == 1
assert deterministic_snapshots[0].snapshot["weather"] == {
"city": "SF",
"temp": 14,
"conditions": "foggy",
}
assert "__ag_ui_tool_result_display__" not in str(deterministic_snapshots[0].snapshot)
assert "__ag_ui_tool_result_state__" not in str(deterministic_snapshots[0].snapshot)
@@ -0,0 +1,90 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the generative UI (workflow-as-agent) scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import WorkflowBuilder, WorkflowContext, executor
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkWorkflow
async def _run(wrapper: AgentFrameworkWorkflow, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in wrapper.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-gen-ui-agent",
"run_id": "run-gen-ui-agent",
"messages": [{"role": "user", "content": "Generate a UI"}],
}
# ── Golden stream tests ──
async def test_workflow_agent_golden_sequence() -> None:
"""Workflow-as-agent: emits step events and text content."""
@executor(id="generator")
async def generator(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("Here is your generated UI content!")
workflow = WorkflowBuilder(start_executor=generator).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_text_messages_balanced()
# Should have step events for the executor
stream.assert_has_type("STEP_STARTED")
stream.assert_has_type("STEP_FINISHED")
# Should have text message content
stream.assert_has_type("TEXT_MESSAGE_CONTENT")
async def test_workflow_agent_step_names_match() -> None:
"""Step started/finished events reference the executor name."""
@executor(id="my_executor")
async def my_executor(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("Done!")
workflow = WorkflowBuilder(start_executor=my_executor).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, PAYLOAD)
started = [e for e in stream.get("STEP_STARTED") if getattr(e, "step_name", "") == "my_executor"]
finished = [e for e in stream.get("STEP_FINISHED") if getattr(e, "step_name", "") == "my_executor"]
assert started, "Expected STEP_STARTED for 'my_executor'"
assert finished, "Expected STEP_FINISHED for 'my_executor'"
async def test_workflow_agent_ordered_events() -> None:
"""Workflow events follow expected ordering: RUN_STARTED → STEP_STARTED → content → STEP_FINISHED → RUN_FINISHED."""
@executor(id="my_step")
async def my_step(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("Generated content")
workflow = WorkflowBuilder(start_executor=my_step).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, PAYLOAD)
stream.assert_ordered_types(
[
"RUN_STARTED",
"STEP_STARTED",
"TEXT_MESSAGE_START",
"TEXT_MESSAGE_CONTENT",
"STEP_FINISHED",
"TEXT_MESSAGE_END",
"RUN_FINISHED",
]
)
@@ -0,0 +1,135 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the client-side (declaration-only) tools scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
return AgentFrameworkAgent(agent=stub, **kwargs)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-gen-ui-tool",
"run_id": "run-gen-ui-tool",
"messages": [{"role": "user", "content": "Show me a chart"}],
"tools": [
{
"type": "function",
"function": {
"name": "render_chart",
"description": "Render a chart in the UI",
"parameters": {
"type": "object",
"properties": {"data": {"type": "array"}},
},
},
}
],
}
# ── Golden stream tests ──
async def test_declaration_only_tool_golden_sequence() -> None:
"""Declaration-only tool: TOOL_CALL_START/ARGS emitted, TOOL_CALL_END at stream end."""
# The LLM calls a client-side tool (no server-side execution)
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="render_chart",
call_id="call-chart",
arguments='{"data": [1, 2, 3]}',
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
# Tool call start and args should be present
stream.assert_has_type("TOOL_CALL_START")
stream.assert_has_type("TOOL_CALL_ARGS")
# TOOL_CALL_END should be emitted (via get_pending_without_end)
stream.assert_has_type("TOOL_CALL_END")
stream.assert_tool_calls_balanced()
async def test_declaration_only_tool_no_tool_call_result() -> None:
"""Declaration-only tools should NOT produce TOOL_CALL_RESULT events."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="render_chart",
call_id="call-chart",
arguments='{"data": [1, 2, 3]}',
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
assert "TOOL_CALL_RESULT" not in stream.types(), "Declaration-only tools should not have TOOL_CALL_RESULT"
async def test_declaration_only_tool_text_messages_balanced() -> None:
"""Text messages remain balanced even with declaration-only tools."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="render_chart",
call_id="call-chart",
arguments='{"data": [1, 2, 3]}',
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_text_messages_balanced()
async def test_declaration_only_tool_messages_snapshot() -> None:
"""MessagesSnapshotEvent includes the tool call for declaration-only tools."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="render_chart",
call_id="call-chart",
arguments='{"data": [1, 2, 3]}',
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_has_type("MESSAGES_SNAPSHOT")
@@ -0,0 +1,194 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the HITL (human-in-the-loop) approval scenario."""
from __future__ import annotations
import json
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent
PREDICT_CONFIG = {
"tasks": {
"tool": "generate_task_steps",
"tool_argument": "steps",
}
}
STATE_SCHEMA = {
"tasks": {"type": "array", "items": {"type": "object"}},
}
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
return AgentFrameworkAgent(
agent=stub,
state_schema=STATE_SCHEMA,
predict_state_config=PREDICT_CONFIG,
require_confirmation=True,
**kwargs,
)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
STEPS = [
{"description": "Step 1: Plan", "status": "enabled"},
{"description": "Step 2: Execute", "status": "enabled"},
]
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-hitl",
"run_id": "run-hitl",
"messages": [{"role": "user", "content": "Plan my tasks"}],
"state": {"tasks": []},
}
# ── Turn 1: Tool call → confirm_changes → interrupt ──
async def test_hitl_turn1_golden_sequence() -> None:
"""Turn 1 emits tool call, confirm_changes, and finishes with interrupt."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="generate_task_steps",
call_id="call-steps",
arguments=json.dumps({"steps": STEPS}),
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
# Should have: tool call start/args/end for the primary tool,
# then TOOL_CALL_END, STATE_SNAPSHOT, confirm_changes cycle
stream.assert_bookends()
stream.assert_no_run_error()
# confirm_changes tool call should be present
tool_starts = stream.get("TOOL_CALL_START")
tool_names = [getattr(s, "tool_call_name", None) for s in tool_starts]
assert "generate_task_steps" in tool_names
assert "confirm_changes" in tool_names
# RUN_FINISHED should have interrupt metadata
interrupt = stream.run_finished_interrupts()
assert len(interrupt) > 0
async def test_hitl_turn1_tool_calls_balanced() -> None:
"""All tool calls in turn 1 (primary + confirm_changes) are balanced."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="generate_task_steps",
call_id="call-steps",
arguments=json.dumps({"steps": STEPS}),
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_tool_calls_balanced()
async def test_hitl_turn1_text_messages_balanced() -> None:
"""Text messages are balanced even in the approval flow."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="generate_task_steps",
call_id="call-steps",
arguments=json.dumps({"steps": STEPS}),
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_text_messages_balanced()
# ── Turn 2: Resume with approval → confirmation message → no interrupt ──
async def test_hitl_turn2_resume_with_approval() -> None:
"""Resuming with confirm_changes result emits confirmation text and finishes cleanly."""
# Turn 2: user sends confirm_changes result as resume
# The agent wrapper sees a confirm_changes response and emits a confirmation message
confirm_result = json.dumps(
{
"accepted": True,
"steps": STEPS,
}
)
# Build payload with resume containing the approval
# For confirm_changes, the messages should include the tool result
payload: dict[str, Any] = {
"thread_id": "thread-hitl",
"run_id": "run-hitl-2",
"messages": [
{"role": "user", "content": "Plan my tasks"},
{
"role": "assistant",
"tool_calls": [
{
"id": "confirm-id-1",
"type": "function",
"function": {"name": "confirm_changes", "arguments": json.dumps({"steps": STEPS})},
}
],
},
{
"role": "tool",
"toolCallId": "confirm-id-1",
"content": confirm_result,
},
],
"state": {"tasks": []},
}
# In turn 2, the agent sees the confirm_changes result and emits a confirmation text
updates = [
AgentResponseUpdate(
contents=[Content.from_text(text="Tasks confirmed!")],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, payload)
stream.assert_bookends()
stream.assert_text_messages_balanced()
stream.assert_no_run_error()
# Should have text message content (the confirmation message)
text_events = stream.get("TEXT_MESSAGE_CONTENT")
assert text_events, "Expected confirmation text message"
# RUN_FINISHED should NOT have interrupt (approval completed)
finished = stream.last("RUN_FINISHED")
dumped = finished.model_dump(by_alias=True, exclude_none=True)
assert "outcome" not in dumped, f"Expected no interrupt after approval, got {dumped.get('outcome')}"
@@ -0,0 +1,130 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the predictive state scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkAgent
PREDICT_CONFIG = {
"document": {
"tool": "update_document",
"tool_argument": "content",
}
}
STATE_SCHEMA = {
"document": {"type": "string"},
}
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(updates=updates)
return AgentFrameworkAgent(
agent=stub,
state_schema=STATE_SCHEMA,
predict_state_config=PREDICT_CONFIG,
require_confirmation=False,
**kwargs,
)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-predict",
"run_id": "run-predict",
"messages": [{"role": "user", "content": "Write a document"}],
"state": {"document": ""},
}
# ── Golden stream tests ──
async def test_predictive_state_emits_deltas_during_tool_args() -> None:
"""STATE_DELTA events are emitted as tool arguments stream in."""
updates = [
AgentResponseUpdate(
contents=[Content.from_function_call(name="update_document", call_id="call-1", arguments="")],
role="assistant",
),
AgentResponseUpdate(
contents=[
Content.from_function_call(name="update_document", call_id="call-1", arguments='{"content": "Hello')
],
role="assistant",
),
AgentResponseUpdate(
contents=[Content.from_function_call(name="update_document", call_id="call-1", arguments=' world"}')],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
# PredictState custom event should be present
custom_events = stream.get("CUSTOM")
predict_events = [e for e in custom_events if getattr(e, "name", None) == "PredictState"]
assert predict_events, "Expected PredictState custom event"
# STATE_DELTA events should be emitted during tool arg streaming
assert "STATE_DELTA" in stream.types(), "Expected STATE_DELTA events during predictive streaming"
async def test_predictive_state_snapshot_after_tool_end() -> None:
"""STATE_SNAPSHOT is emitted when a predictive tool completes (no confirmation)."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(
name="update_document", call_id="call-1", arguments='{"content": "Final text"}'
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
# Should have initial state snapshot + updated snapshot after tool completion
snapshots = stream.get("STATE_SNAPSHOT")
assert len(snapshots) >= 1, "Expected at least one STATE_SNAPSHOT"
async def test_predictive_state_ordered_events() -> None:
"""Event ordering: RUN_STARTED → PredictState → STATE_SNAPSHOT → TOOL_CALL_* → STATE_SNAPSHOT → RUN_FINISHED."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_function_call(name="update_document", call_id="call-1", arguments='{"content": "doc"}')
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_ordered_types(
[
"RUN_STARTED",
"CUSTOM", # PredictState
"STATE_SNAPSHOT", # Initial state
"TOOL_CALL_START",
"TOOL_CALL_ARGS",
"RUN_FINISHED",
]
)
@@ -0,0 +1,110 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the shared state (structured output) scenario."""
from __future__ import annotations
from typing import Any
from agent_framework import AgentResponseUpdate, Content
from conftest import StubAgent # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from pydantic import BaseModel
from agent_framework_ag_ui import AgentFrameworkAgent
class RecipeState(BaseModel):
recipe_title: str = ""
ingredients: list[str] = []
message: str = ""
def _build_agent(updates: list[AgentResponseUpdate], **kwargs: Any) -> AgentFrameworkAgent:
stub = StubAgent(
updates=updates,
default_options={"tools": None, "response_format": RecipeState},
)
return AgentFrameworkAgent(
agent=stub,
state_schema={
"recipe_title": {"type": "string"},
"ingredients": {"type": "array", "items": {"type": "string"}},
},
**kwargs,
)
async def _run(agent: AgentFrameworkAgent, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
PAYLOAD: dict[str, Any] = {
"thread_id": "thread-state",
"run_id": "run-state",
"messages": [{"role": "user", "content": "Give me a pasta recipe"}],
"state": {"recipe_title": "", "ingredients": []},
}
# ── Golden stream tests ──
async def test_shared_state_emits_state_snapshot() -> None:
"""Structured output agent emits STATE_SNAPSHOT with parsed model fields."""
# The structured output agent gets a response that the framework parses as RecipeState
updates = [
AgentResponseUpdate(
contents=[
Content.from_text(
text='{"recipe_title": "Pasta Carbonara", "ingredients": ["pasta", "eggs", "cheese"], "message": "Here is your recipe!"}'
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
stream.assert_bookends()
stream.assert_no_run_error()
# Should have STATE_SNAPSHOT with the initial state at minimum
stream.assert_has_type("STATE_SNAPSHOT")
async def test_shared_state_initial_snapshot_on_first_update() -> None:
"""When state_schema and state are provided, initial STATE_SNAPSHOT is emitted after RUN_STARTED."""
updates = [
AgentResponseUpdate(
contents=[Content.from_text(text='{"recipe_title": "Test", "ingredients": [], "message": "hi"}')],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
# RUN_STARTED should be followed by STATE_SNAPSHOT (initial state)
stream.assert_ordered_types(["RUN_STARTED", "STATE_SNAPSHOT"])
async def test_shared_state_text_emitted_from_message_field() -> None:
"""Structured output's 'message' field is emitted as text message events."""
updates = [
AgentResponseUpdate(
contents=[
Content.from_text(
text='{"recipe_title": "Pasta", "ingredients": ["pasta"], "message": "Enjoy your pasta!"}'
)
],
role="assistant",
),
]
agent = _build_agent(updates)
stream = await _run(agent, PAYLOAD)
# Text should be emitted from the message field
text_contents = stream.get("TEXT_MESSAGE_CONTENT")
if text_contents:
combined = "".join(getattr(e, "delta", "") for e in text_contents)
assert "Enjoy your pasta!" in combined
@@ -0,0 +1,207 @@
# Copyright (c) Microsoft. All rights reserved.
"""Golden event-stream tests for the workflow HITL (subgraphs) scenario.
Extends the existing test_subgraphs_example_agent.py with EventStream assertions
on full event ordering, balancing, and interrupt structure.
"""
from __future__ import annotations
import json
from typing import Any
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui_examples.agents.subgraphs_agent import subgraphs_agent
async def _run(agent: Any, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in agent.run(payload)])
# ── Turn 1: Initial request → flight interrupt ──
async def test_subgraphs_turn1_golden_bookends() -> None:
"""Turn 1 starts with RUN_STARTED and ends with RUN_FINISHED."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-1",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip to San Francisco"}],
},
)
stream.assert_bookends()
async def test_subgraphs_turn1_no_errors() -> None:
"""Turn 1 completes without errors."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-2",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip"}],
},
)
stream.assert_no_run_error()
async def test_subgraphs_turn1_has_step_events() -> None:
"""Turn 1 emits STEP_STARTED and STEP_FINISHED for workflow executors."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-3",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip"}],
},
)
stream.assert_has_type("STEP_STARTED")
stream.assert_has_type("STEP_FINISHED")
async def test_subgraphs_turn1_interrupt_structure() -> None:
"""Turn 1 RUN_FINISHED carries flight interrupt with correct structure."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-4",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip to SF"}],
},
)
interrupt = stream.run_finished_interrupts()
assert len(interrupt) > 0
interrupt_value = stream.interrupt_metadata_value(interrupt[0])
assert interrupt_value["agent"] == "flights"
assert len(interrupt_value["options"]) == 2
async def test_subgraphs_turn1_text_messages_balanced() -> None:
"""All text messages in turn 1 are properly balanced."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-5",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip"}],
},
)
stream.assert_text_messages_balanced()
async def test_subgraphs_turn1_ordered_flow() -> None:
"""Turn 1 event ordering: RUN_STARTED → STATE_SNAPSHOT → STEP_* → TOOL_CALL_* → RUN_FINISHED."""
agent = subgraphs_agent()
stream = await _run(
agent,
{
"thread_id": "thread-sub-golden-6",
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip"}],
},
)
stream.assert_ordered_types(
[
"RUN_STARTED",
"STATE_SNAPSHOT",
"STEP_STARTED",
"RUN_FINISHED",
]
)
# ── Multi-turn: Flight selection → hotel interrupt → completion ──
async def test_subgraphs_full_flow_event_ordering() -> None:
"""Complete 3-turn flow maintains proper event ordering throughout."""
agent = subgraphs_agent()
thread_id = "thread-sub-golden-full"
# Turn 1
stream1 = await _run(
agent,
{
"thread_id": thread_id,
"run_id": "run-1",
"messages": [{"role": "user", "content": "Plan a trip to SF from Amsterdam"}],
},
)
stream1.assert_bookends()
stream1.assert_no_run_error()
# Extract flight interrupt
interrupt1 = stream1.run_finished_interrupts()[0]
# Turn 2: Select flight
stream2 = await _run(
agent,
{
"thread_id": thread_id,
"run_id": "run-2",
"resume": {
"interrupts": [
{
"id": interrupt1["id"],
"value": json.dumps(
{
"airline": "United",
"departure": "Amsterdam (AMS)",
"arrival": "San Francisco (SFO)",
"price": "$720",
"duration": "12h 15m",
}
),
}
]
},
},
)
stream2.assert_bookends()
stream2.assert_no_run_error()
# Should now have hotel interrupt
interrupt2 = stream2.run_finished_interrupts()
assert stream2.interrupt_metadata_value(interrupt2[0])["agent"] == "hotels"
# Turn 3: Select hotel
stream3 = await _run(
agent,
{
"thread_id": thread_id,
"run_id": "run-3",
"resume": {
"interrupts": [
{
"id": interrupt2[0]["id"],
"value": json.dumps(
{
"name": "The Ritz-Carlton",
"location": "Nob Hill",
"price_per_night": "$550/night",
"rating": "4.8 stars",
}
),
}
]
},
},
)
stream3.assert_bookends()
stream3.assert_no_run_error()
stream3.assert_text_messages_balanced()
# Final turn should not have interrupt
finished3 = stream3.last("RUN_FINISHED")
final_dump = finished3.model_dump(by_alias=True, exclude_none=True)
assert "outcome" not in final_dump, f"Expected no interrupt after completion, got {final_dump.get('outcome')}"
@@ -0,0 +1,945 @@
# Copyright (c) Microsoft. All rights reserved.
"""Comprehensive golden event-stream tests for AgentFrameworkWorkflow.
Covers the full matrix of workflow-specific AG-UI patterns:
- request_info → TOOL_CALL lifecycle and balancing
- Executor step events and activity snapshots
- Text output, dict output, BaseEvent passthrough, AgentResponse output
- Text deduplication across workflow outputs
- Workflow error handling → RUN_ERROR
- Multi-turn interrupt/resume round-trips
- Empty turns with pending requests
- Custom workflow events
- Text message draining on request_info and executor boundaries
"""
import json
from typing import Any, cast
from ag_ui.core import EventType, StateSnapshotEvent
from agent_framework import (
AgentResponse,
Content,
Executor,
Message,
WorkflowBuilder,
WorkflowContext,
WorkflowEvent,
executor,
handler,
response_handler,
)
from event_stream import EventStream # pyrefly: ignore[missing-import] # pyright: ignore[reportMissingImports]
from agent_framework_ag_ui import AgentFrameworkWorkflow
async def _run(wrapper: AgentFrameworkWorkflow, payload: dict[str, Any]) -> EventStream:
return EventStream([event async for event in wrapper.run(payload)])
def _payload(
msg: str = "go",
*,
thread_id: str = "thread-wf",
run_id: str = "run-wf",
**extra: Any,
) -> dict[str, Any]:
return {"thread_id": thread_id, "run_id": run_id, "messages": [{"role": "user", "content": msg}], **extra}
# ──────────────────────────────────────────────────────────────────────
# 1. Basic workflow text output
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_text_output_golden_sequence() -> None:
"""Simple text output: RUN_STARTED → STEP_STARTED → TEXT_* → STEP_FINISHED → TEXT_MESSAGE_END → RUN_FINISHED."""
@executor(id="greeter")
async def greeter(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("Hello from workflow!")
workflow = WorkflowBuilder(start_executor=greeter).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_no_run_error()
stream.assert_text_messages_balanced()
stream.assert_has_type("TEXT_MESSAGE_START")
stream.assert_has_type("TEXT_MESSAGE_CONTENT")
stream.assert_has_type("TEXT_MESSAGE_END")
# Verify actual content
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
assert "Hello from workflow!" in deltas
async def test_workflow_text_output_message_id_consistency() -> None:
"""All text events for a single output share the same message_id."""
@executor(id="echo")
async def echo(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("echo reply")
workflow = WorkflowBuilder(start_executor=echo).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_message_ids_consistent()
# ──────────────────────────────────────────────────────────────────────
# 2. Executor step events and activity snapshots
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_executor_lifecycle_events() -> None:
"""Executor invocation produces STEP_STARTED, ACTIVITY_SNAPSHOT, STEP_FINISHED."""
@executor(id="worker")
async def worker(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("done")
workflow = WorkflowBuilder(start_executor=worker).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
# Step events with executor ID
started = [e for e in stream.get("STEP_STARTED") if getattr(e, "step_name", "") == "worker"]
finished = [e for e in stream.get("STEP_FINISHED") if getattr(e, "step_name", "") == "worker"]
assert started, "Expected STEP_STARTED for 'worker'"
assert finished, "Expected STEP_FINISHED for 'worker'"
# Activity snapshots
activities = stream.get("ACTIVITY_SNAPSHOT")
assert activities, "Expected ACTIVITY_SNAPSHOT events"
# Check one of them has executor payload
executor_activities = [a for a in activities if getattr(a, "activity_type", None) == "executor"]
assert executor_activities, "Expected executor-type activity snapshots"
async def test_workflow_executor_step_ordering() -> None:
"""STEP_STARTED comes before content, STEP_FINISHED comes after."""
@executor(id="orderer")
async def orderer(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("ordered output")
workflow = WorkflowBuilder(start_executor=orderer).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_ordered_types(
[
"RUN_STARTED",
"STEP_STARTED",
"TEXT_MESSAGE_START",
"TEXT_MESSAGE_CONTENT",
"STEP_FINISHED",
"RUN_FINISHED",
]
)
# ──────────────────────────────────────────────────────────────────────
# 3. Dict output → CUSTOM workflow_output
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_dict_output_maps_to_custom_event() -> None:
"""Non-chat dict output is emitted as CUSTOM workflow_output event."""
@executor(id="structured")
async def structured(message: Any, ctx: WorkflowContext[Any, dict[str, int]]) -> None:
await ctx.yield_output({"count": 42, "status": 1})
workflow = WorkflowBuilder(start_executor=structured).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_no_run_error()
customs = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "workflow_output"]
assert len(customs) == 1
assert customs[0].value == {"count": 42, "status": 1}
# Should NOT have TEXT_MESSAGE events for dict output
assert "TEXT_MESSAGE_CONTENT" not in stream.types()
# ──────────────────────────────────────────────────────────────────────
# 4. BaseEvent passthrough
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_base_event_passthrough() -> None:
"""AG-UI BaseEvent outputs are yielded directly, not wrapped."""
@executor(id="stateful")
async def stateful(message: Any, ctx: WorkflowContext[Any, StateSnapshotEvent]) -> None:
await ctx.yield_output(StateSnapshotEvent(type=EventType.STATE_SNAPSHOT, snapshot={"active_agent": "flights"}))
workflow = WorkflowBuilder(start_executor=stateful).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
snapshots = stream.get("STATE_SNAPSHOT")
assert len(snapshots) == 1
assert snapshots[0].snapshot["active_agent"] == "flights"
# ──────────────────────────────────────────────────────────────────────
# 5. AgentResponse output (conversation payload)
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_agent_response_output_extracts_latest_assistant() -> None:
"""AgentResponse output uses only the latest assistant message, not full history."""
@executor(id="responder")
async def responder(message: Any, ctx: WorkflowContext[Any, AgentResponse]) -> None:
response = AgentResponse(
messages=[
Message(role="user", contents=[Content.from_text("My order is damaged")]),
Message(role="assistant", contents=[Content.from_text("I'll process your replacement.")]),
]
)
await ctx.yield_output(response)
workflow = WorkflowBuilder(start_executor=responder).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_text_messages_balanced()
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
assert deltas == ["I'll process your replacement."]
# ──────────────────────────────────────────────────────────────────────
# 6. Custom workflow events
# ──────────────────────────────────────────────────────────────────────
class ProgressEvent(WorkflowEvent):
"""Custom workflow event for testing CUSTOM event mapping."""
def __init__(self, progress: int) -> None:
super().__init__(cast(Any, "custom_progress"), data={"progress": progress})
async def test_workflow_custom_events() -> None:
"""Custom workflow events are mapped to CUSTOM AG-UI events."""
@executor(id="progress_tracker")
async def progress_tracker(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.add_event(ProgressEvent(25))
await ctx.yield_output("In progress...")
await ctx.add_event(ProgressEvent(100))
workflow = WorkflowBuilder(start_executor=progress_tracker).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_no_run_error()
progress_events = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "custom_progress"]
assert len(progress_events) == 2
assert progress_events[0].value == {"progress": 25}
assert progress_events[1].value == {"progress": 100}
# ──────────────────────────────────────────────────────────────────────
# 7. request_info → TOOL_CALL lifecycle
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_request_info_tool_call_lifecycle() -> None:
"""request_info emits TOOL_CALL_START/ARGS/END cycle plus CUSTOM request_info."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info("Need approval", str, request_id="req-1")
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_no_run_error()
# Tool call lifecycle
stream.assert_ordered_types(
[
"RUN_STARTED",
"TOOL_CALL_START",
"TOOL_CALL_ARGS",
"TOOL_CALL_END",
"CUSTOM", # request_info
"RUN_FINISHED",
]
)
# Verify tool call details
start = stream.first("TOOL_CALL_START")
assert start.tool_call_id == "req-1"
assert start.tool_call_name == "request_info"
# TOOL_CALL_ARGS should contain the request payload
args = stream.first("TOOL_CALL_ARGS")
assert args.tool_call_id == "req-1"
parsed_args = json.loads(args.delta)
assert parsed_args["request_id"] == "req-1"
# Tool calls should be balanced
stream.assert_tool_calls_balanced()
async def test_workflow_request_info_interrupt_in_run_finished() -> None:
"""request_info populates RUN_FINISHED.outcome.interrupts with the request metadata."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info(
{"message": "Choose a flight", "options": [{"airline": "KLM"}], "agent": "flights"},
dict,
request_id="flights-choice",
)
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
interrupt = stream.run_finished_interrupts()
assert len(interrupt) == 1
assert interrupt[0]["id"] == "flights-choice"
assert stream.interrupt_metadata_value(interrupt[0])["agent"] == "flights"
async def test_workflow_request_info_emits_interrupt_card_event() -> None:
"""request_info with dict data emits a WorkflowInterruptEvent custom event."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info(
{"message": "Pick one", "options": ["A", "B"]},
dict,
request_id="pick-1",
)
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
interrupt_cards = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "WorkflowInterruptEvent"]
assert interrupt_cards, "Expected WorkflowInterruptEvent custom event"
# ──────────────────────────────────────────────────────────────────────
# 8. Text message draining on request_info boundary
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_text_drained_before_request_info() -> None:
"""Open text message is closed (TEXT_MESSAGE_END) before request_info tool calls begin."""
@executor(id="text_then_request")
async def text_then_request(message: Any, ctx: WorkflowContext) -> None:
await ctx.yield_output("Please confirm this action.") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
await ctx.request_info("Need approval", str, request_id="approval-1")
workflow = WorkflowBuilder(start_executor=text_then_request).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_text_messages_balanced()
stream.assert_tool_calls_balanced()
# TEXT_MESSAGE_END must appear before TOOL_CALL_START
types = stream.types()
text_end_idx = types.index("TEXT_MESSAGE_END")
tool_start_idx = types.index("TOOL_CALL_START")
assert text_end_idx < tool_start_idx, (
f"TEXT_MESSAGE_END (idx={text_end_idx}) must come before TOOL_CALL_START (idx={tool_start_idx})"
)
# ──────────────────────────────────────────────────────────────────────
# 9. Text deduplication
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_skips_duplicate_text_from_snapshot() -> None:
"""Duplicate text from AgentResponse snapshot is not re-emitted."""
@executor(id="deduper")
async def deduper(message: Any, ctx: WorkflowContext[Any, Any]) -> None:
text = "Order processed successfully."
await ctx.yield_output(text)
# Snapshot repeats the same text
await ctx.yield_output(
AgentResponse(
messages=[
Message(role="user", contents=[Content.from_text("process order")]),
Message(role="assistant", contents=[Content.from_text(text)]),
]
)
)
workflow = WorkflowBuilder(start_executor=deduper).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_text_messages_balanced()
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
# Text should appear only once
assert deltas == ["Order processed successfully."]
async def test_workflow_skips_consecutive_duplicate_outputs() -> None:
"""Consecutive identical text outputs are deduplicated."""
@executor(id="repeater")
async def repeater(message: Any, ctx: WorkflowContext[Any, Any]) -> None:
text = "Done!"
await ctx.yield_output(text)
await ctx.yield_output(text)
workflow = WorkflowBuilder(start_executor=repeater).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_text_messages_balanced()
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
assert deltas == ["Done!"]
async def test_workflow_emits_distinct_consecutive_outputs() -> None:
"""Distinct text outputs are all emitted, not incorrectly deduplicated."""
@executor(id="multisayer")
async def multisayer(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("First part. ")
await ctx.yield_output("Second part.")
workflow = WorkflowBuilder(start_executor=multisayer).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_text_messages_balanced()
deltas = [e.delta for e in stream.get("TEXT_MESSAGE_CONTENT")]
assert deltas == ["First part. ", "Second part."]
# ──────────────────────────────────────────────────────────────────────
# 10. Workflow error handling → RUN_ERROR
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_error_emits_run_error_event() -> None:
"""Exceptions during workflow streaming produce RUN_ERROR events."""
class FailingWorkflow:
def run(self, **kwargs: Any):
async def _stream():
raise RuntimeError("workflow exploded")
yield # pragma: no cover
return _stream()
wrapper = AgentFrameworkWorkflow(workflow=cast(Any, FailingWorkflow()))
stream = await _run(wrapper, _payload())
# Should still have RUN_STARTED
stream.assert_has_type("RUN_STARTED")
# Should have RUN_ERROR
stream.assert_has_type("RUN_ERROR")
error = stream.first("RUN_ERROR")
assert "workflow exploded" in error.message
async def test_workflow_error_preserves_bookend_structure() -> None:
"""Even on error, RUN_STARTED is the first event."""
class FailingWorkflow:
def run(self, **kwargs: Any):
async def _stream():
raise ValueError("bad input")
yield # pragma: no cover
return _stream()
wrapper = AgentFrameworkWorkflow(workflow=cast(Any, FailingWorkflow()))
stream = await _run(wrapper, _payload())
types = stream.types()
assert types[0] == "RUN_STARTED"
assert "RUN_ERROR" in types
# ──────────────────────────────────────────────────────────────────────
# 11. Multi-turn request_info interrupt/resume
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_interrupt_resume_round_trip() -> None:
"""Turn 1: request_info → interrupt. Turn 2: resume → completion."""
class RequesterExecutor(Executor):
def __init__(self) -> None:
super().__init__(id="requester")
@handler
async def start(self, message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info("Choose an option", str, request_id="choice-1")
@response_handler
async def handle_choice(self, original: str, response: str, ctx: WorkflowContext) -> None:
await ctx.yield_output(f"You chose: {response}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
workflow = WorkflowBuilder(start_executor=RequesterExecutor()).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1
stream1 = await _run(wrapper, _payload(thread_id="thread-resume", run_id="run-1"))
stream1.assert_bookends()
stream1.assert_no_run_error()
stream1.assert_tool_calls_balanced()
interrupt1 = stream1.run_finished_interrupts()
assert interrupt1, "Expected interrupt"
assert interrupt1[0]["id"] == "choice-1"
# Turn 2: resume
stream2 = await _run(
wrapper,
{
"thread_id": "thread-resume",
"run_id": "run-2",
"messages": [],
"resume": {"interrupts": [{"id": "choice-1", "value": "Option A"}]},
},
)
stream2.assert_has_run_lifecycle()
stream2.assert_no_run_error()
stream2.assert_text_messages_balanced()
# Should have the response text
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
assert any("Option A" in d for d in deltas), f"Expected 'Option A' in deltas: {deltas}"
# No interrupt after resume
finished2 = stream2.last("RUN_FINISHED")
dumped2 = finished2.model_dump(by_alias=True, exclude_none=True)
assert "outcome" not in dumped2
async def test_workflow_forwarded_props_resume() -> None:
"""forwarded_props.command.resume should resume with canonical resume entries."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info({"options": [{"name": "A"}]}, dict, request_id="pick")
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1
await _run(wrapper, _payload(thread_id="thread-fwd", run_id="run-1"))
# Turn 2 via forwarded_props
stream2 = await _run(
wrapper,
{
"thread_id": "thread-fwd",
"run_id": "run-2",
"messages": [],
"forwarded_props": {
"command": {"resume": [{"interruptId": "pick", "status": "resolved", "payload": {"name": "A"}}]}
},
},
)
stream2.assert_bookends()
stream2.assert_no_run_error()
finished = stream2.last("RUN_FINISHED")
assert "outcome" not in finished.model_dump(by_alias=True, exclude_none=True)
# ──────────────────────────────────────────────────────────────────────
# 12. Empty turns with pending requests
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_empty_turn_with_pending_request_emits_run_error() -> None:
"""An empty turn with a pending request must provide canonical resume entries."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info({"prompt": "choose"}, dict, request_id="pick-one")
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1: trigger the request
await _run(wrapper, _payload(thread_id="thread-empty", run_id="run-1"))
# Turn 2: empty messages, no resume
stream2 = await _run(
wrapper,
{
"thread_id": "thread-empty",
"run_id": "run-2",
"messages": [],
},
)
stream2.assert_has_type("RUN_STARTED")
run_error = stream2.last("RUN_ERROR")
assert run_error.code == "WORKFLOW_RESUME_REQUIRED"
async def test_workflow_empty_turn_no_pending_requests() -> None:
"""Empty turn with no pending requests produces clean bookends."""
@executor(id="noop")
async def noop(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output("done")
workflow = WorkflowBuilder(start_executor=noop).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Run once to completion
await _run(wrapper, _payload(thread_id="thread-empty-clean", run_id="run-1"))
# Empty turn
stream2 = await _run(
wrapper,
{
"thread_id": "thread-empty-clean",
"run_id": "run-2",
"messages": [],
},
)
stream2.assert_bookends()
stream2.assert_no_run_error()
# ──────────────────────────────────────────────────────────────────────
# 13. Usage content as CUSTOM event
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_usage_output_maps_to_custom_event() -> None:
"""Usage Content outputs are surfaced as custom usage events."""
@executor(id="usage_reporter")
async def usage_reporter(message: Any, ctx: WorkflowContext[Any, Content]) -> None:
await ctx.yield_output(
Content.from_usage({"input_token_count": 100, "output_token_count": 50, "total_token_count": 150})
)
workflow = WorkflowBuilder(start_executor=usage_reporter).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
stream = await _run(wrapper, _payload())
stream.assert_bookends()
stream.assert_no_run_error()
usage_events = [e for e in stream.get("CUSTOM") if getattr(e, "name", None) == "usage"]
assert len(usage_events) == 1
assert usage_events[0].value["input_token_count"] == 100
assert usage_events[0].value["total_token_count"] == 150
# ──────────────────────────────────────────────────────────────────────
# 14. Approval flow (Content-based request_info)
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_approval_flow_round_trip() -> None:
"""function_approval_request via request_info, then resume with approval response."""
class ApprovalExecutor(Executor):
def __init__(self) -> None:
super().__init__(id="approval_exec")
@handler
async def start(self, message: Any, ctx: WorkflowContext) -> None:
function_call = Content.from_function_call(
call_id="refund-call",
name="submit_refund",
arguments={"order_id": "12345", "amount": "$89.99"},
)
approval_request = Content.from_function_approval_request(id="approval-1", function_call=function_call)
await ctx.request_info(approval_request, Content, request_id="approval-1")
@response_handler
async def handle_approval(self, original_request: Content, response: Content, ctx: WorkflowContext) -> None:
status = "approved" if bool(response.approved) else "rejected"
await ctx.yield_output(f"Refund {status}.") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
workflow = WorkflowBuilder(start_executor=ApprovalExecutor()).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1: request approval
stream1 = await _run(wrapper, _payload(thread_id="thread-approval", run_id="run-1"))
stream1.assert_bookends()
stream1.assert_no_run_error()
interrupt1 = stream1.run_finished_interrupts()
assert interrupt1, "Expected approval interrupt"
interrupt_value = stream1.interrupt_metadata_value(interrupt1[0])
# Turn 2: approve
stream2 = await _run(
wrapper,
{
"thread_id": "thread-approval",
"run_id": "run-2",
"messages": [],
"resume": {
"interrupts": [
{
"id": "approval-1",
"value": {
"type": "function_approval_response",
"approved": True,
"id": interrupt_value.get("id", "approval-1"),
"function_call": interrupt_value.get("function_call"),
},
}
]
},
},
)
stream2.assert_has_run_lifecycle()
stream2.assert_no_run_error()
stream2.assert_text_messages_balanced()
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
assert any("approved" in d for d in deltas)
# No more interrupt
finished2 = stream2.last("RUN_FINISHED")
assert "outcome" not in finished2.model_dump(by_alias=True, exclude_none=True)
# ──────────────────────────────────────────────────────────────────────
# 15. Message list request/response coercion
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_message_list_resume() -> None:
"""Resume with list[Message] payload coerces correctly into workflow response."""
class MessageRequestExecutor(Executor):
def __init__(self) -> None:
super().__init__(id="msg_request")
@handler
async def start(self, message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info({"prompt": "Need follow-up"}, list[Message], request_id="handoff")
@response_handler
async def handle_input(self, original: dict, response: list[Message], ctx: WorkflowContext) -> None:
user_text = response[0].text if response else ""
await ctx.yield_output(f"Got: {user_text}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
workflow = WorkflowBuilder(start_executor=MessageRequestExecutor()).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1
await _run(wrapper, _payload(thread_id="thread-msg", run_id="run-1"))
# Turn 2: resume with message list
stream2 = await _run(
wrapper,
{
"thread_id": "thread-msg",
"run_id": "run-2",
"messages": [],
"resume": {
"interrupts": [
{
"id": "handoff",
"value": [
{"role": "user", "contents": [{"type": "text", "text": "Ship a replacement"}]},
],
}
]
},
},
)
stream2.assert_has_run_lifecycle()
stream2.assert_no_run_error()
stream2.assert_text_messages_balanced()
deltas = [e.delta for e in stream2.get("TEXT_MESSAGE_CONTENT")]
assert any("replacement" in d for d in deltas)
# ──────────────────────────────────────────────────────────────────────
# 16. Plain text follow-up does NOT infer interrupt response
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_plain_text_does_not_resume_pending_dict_request() -> None:
"""Plain text user follow-up should fail instead of being coerced into a dict response."""
@executor(id="requester")
async def requester(message: Any, ctx: WorkflowContext) -> None:
await ctx.request_info(
{"message": "Choose a flight", "options": [{"airline": "KLM"}], "agent": "flights"},
dict,
request_id="flights-choice",
)
workflow = WorkflowBuilder(start_executor=requester).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1
await _run(wrapper, _payload(thread_id="thread-nocoerce", run_id="run-1"))
# Turn 2: plain text follow-up with request_info tool call in history
stream2 = await _run(
wrapper,
{
"thread_id": "thread-nocoerce",
"run_id": "run-2",
"messages": [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "flights-choice",
"type": "function",
"function": {"name": "request_info", "arguments": "{}"},
}
],
},
{"role": "user", "content": "I prefer KLM please"},
],
},
)
stream2.assert_has_type("RUN_STARTED")
run_error = stream2.last("RUN_ERROR")
assert run_error.code == "WORKFLOW_RESUME_REQUIRED"
# ──────────────────────────────────────────────────────────────────────
# 17. Workflow factory (thread-scoped workflows)
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_factory_thread_scoping() -> None:
"""workflow_factory creates separate workflow instances per thread_id."""
def make_workflow(thread_id: str):
@executor(id="echo")
async def echo(message: Any, ctx: WorkflowContext[Any, str]) -> None:
await ctx.yield_output(f"Thread: {thread_id}")
return WorkflowBuilder(start_executor=echo).build()
wrapper = AgentFrameworkWorkflow(workflow_factory=make_workflow)
stream_a = await _run(wrapper, _payload(thread_id="thread-a", run_id="run-a"))
stream_b = await _run(wrapper, _payload(thread_id="thread-b", run_id="run-b"))
stream_a.assert_bookends()
stream_b.assert_bookends()
deltas_a = [e.delta for e in stream_a.get("TEXT_MESSAGE_CONTENT")]
deltas_b = [e.delta for e in stream_b.get("TEXT_MESSAGE_CONTENT")]
assert any("thread-a" in d for d in deltas_a)
assert any("thread-b" in d for d in deltas_b)
# ──────────────────────────────────────────────────────────────────────
# 18. Multiple request_info calls in sequence
# ──────────────────────────────────────────────────────────────────────
async def test_workflow_sequential_request_info_interrupts() -> None:
"""Two chained executors each requesting info: first triggers interrupt, resume, then second triggers interrupt.
This mirrors the subgraphs_agent pattern where separate executors handle sequential interactions.
"""
class NameRequester(Executor):
def __init__(self) -> None:
super().__init__(id="name_requester")
@handler
async def start(self, message: Any, ctx: WorkflowContext[str]) -> None:
await ctx.request_info("What's your name?", str, request_id="name-req")
@response_handler
async def handle_name(self, original: str, response: str, ctx: WorkflowContext[str]) -> None:
await ctx.send_message(response)
class DestRequester(Executor):
def __init__(self) -> None:
super().__init__(id="dest_requester")
@handler
async def start(self, message: str, ctx: WorkflowContext[str]) -> None:
self._name = message
await ctx.request_info("Where to?", str, request_id="dest-req")
@response_handler
async def handle_dest(self, original: str, response: str, ctx: WorkflowContext[str]) -> None:
await ctx.yield_output(f"Booking for {self._name} to {response}") # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type]
name_requester = NameRequester()
dest_requester = DestRequester()
workflow = WorkflowBuilder(start_executor=name_requester).add_chain([name_requester, dest_requester]).build()
wrapper = AgentFrameworkWorkflow(workflow=workflow)
# Turn 1
stream1 = await _run(wrapper, _payload(thread_id="thread-seq", run_id="run-1"))
stream1.assert_bookends()
stream1.assert_tool_calls_balanced()
interrupt1 = stream1.run_finished_interrupts()
assert interrupt1[0]["id"] == "name-req"
# Turn 2: answer name → triggers second executor's request_info
stream2 = await _run(
wrapper,
{
"thread_id": "thread-seq",
"run_id": "run-2",
"messages": [],
"resume": {"interrupts": [{"id": "name-req", "value": "Alice"}]},
},
)
stream2.assert_has_run_lifecycle()
stream2.assert_tool_calls_balanced()
interrupt2 = stream2.run_finished_interrupts()
assert interrupt2[0]["id"] == "dest-req"
# Turn 3: answer destination → completion
stream3 = await _run(
wrapper,
{
"thread_id": "thread-seq",
"run_id": "run-3",
"messages": [],
"resume": {"interrupts": [{"id": "dest-req", "value": "Paris"}]},
},
)
stream3.assert_has_run_lifecycle()
stream3.assert_no_run_error()
stream3.assert_text_messages_balanced()
deltas = [e.delta for e in stream3.get("TEXT_MESSAGE_CONTENT")]
assert any("Alice" in d and "Paris" in d for d in deltas)
assert "outcome" not in stream3.last("RUN_FINISHED").model_dump(by_alias=True, exclude_none=True)