chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
"""Pytest path configuration for ag2 showcase unit tests.
|
||||
|
||||
Ensures ``src/`` (so ``agents.*`` imports resolve) and the integration root
|
||||
(so the ``_shared`` symlink → ``../_shared`` resolves as a package) are on
|
||||
``sys.path``. Mirrors the strands integration's conftest path setup; ag2 tests
|
||||
import the real agent modules (autogen is installed in CI / the image), so no
|
||||
stub modules are installed here.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
_HERE = os.path.dirname(__file__)
|
||||
_PKG_ROOT = os.path.abspath(os.path.join(_HERE, "..", ".."))
|
||||
|
||||
# src/ holds agent_server.py and agents/
|
||||
sys.path.insert(0, os.path.join(_PKG_ROOT, "src"))
|
||||
# project root: the ``_shared`` symlink → ../_shared lives here, plus the
|
||||
# ``tools`` symlink the agent modules rely on.
|
||||
sys.path.insert(0, _PKG_ROOT)
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Red→green tests for the ag2 backend CVDIAG boundary instrumentation.
|
||||
|
||||
Exercises the REAL emit surface — every assertion reads the actual
|
||||
``CVDIAG {<json>}`` lines that ``_shared.cvdiag_bootstrap.emit_cvdiag`` writes
|
||||
to stdout (captured via ``capsys``), driven through the real
|
||||
``CvdiagBackendMiddleware`` and the real ``LlmCallScope`` / agent helpers. No
|
||||
mocks of the emit path.
|
||||
|
||||
What's covered (spec §3 / §5 / §6):
|
||||
* All 11 backend boundaries emit to stdout across the three request shapes
|
||||
that collectively exercise them (happy streaming, aborted stream, raised
|
||||
exception) for synthetic requests with ``CVDIAG_BACKEND_EMITTER=1`` (run at
|
||||
DEBUG tier so the verbose+debug boundaries are permitted).
|
||||
* PII scrub: a synthetic ``sk-test-12345`` in an exception message never
|
||||
appears in the emitted ``backend.error.caught`` JSON.
|
||||
* Heartbeat fires within ~12s of a slow-LLM simulation.
|
||||
* Default-OFF: with the flag unset, NO CVDIAG backend line is emitted.
|
||||
|
||||
RED before instrumentation: ``agents._cvdiag_backend`` does not exist →
|
||||
ImportError; the 11-boundary / heartbeat / scrub assertions cannot pass.
|
||||
GREEN after: every boundary, the scrub, and the heartbeat assert true.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Dict, List
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.responses import StreamingResponse
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from agents._cvdiag_backend import (
|
||||
CvdiagBackendMiddleware,
|
||||
LlmCallScope,
|
||||
_RequestCtx,
|
||||
emit_agent_enter,
|
||||
emit_agent_exit,
|
||||
scrub,
|
||||
)
|
||||
|
||||
# The 11 backend boundaries (spec §5).
|
||||
ALL_BACKEND_BOUNDARIES = {
|
||||
"backend.request.ingress",
|
||||
"backend.agent.enter",
|
||||
"backend.llm.call.start",
|
||||
"backend.llm.call.heartbeat",
|
||||
"backend.llm.call.response",
|
||||
"backend.sse.first_byte",
|
||||
"backend.sse.event",
|
||||
"backend.sse.aborted",
|
||||
"backend.agent.exit",
|
||||
"backend.response.complete",
|
||||
"backend.error.caught",
|
||||
}
|
||||
|
||||
VALID_TEST_ID = "0190a9c0-1a2b-7c3d-8e4f-5a6b7c8d9e0f"
|
||||
|
||||
|
||||
def _parse_cvdiag_lines(captured: str) -> List[Dict]:
|
||||
"""Extract every ``CVDIAG {<json>}`` envelope line from captured stdout."""
|
||||
out: List[Dict] = []
|
||||
for line in captured.splitlines():
|
||||
if line.startswith("CVDIAG {"):
|
||||
out.append(json.loads(line[len("CVDIAG ") :]))
|
||||
return out
|
||||
|
||||
|
||||
def _boundaries(envelopes: List[Dict]) -> set:
|
||||
return {e["boundary"] for e in envelopes}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _debug_tier(monkeypatch):
|
||||
"""Run each test at DEBUG tier so verbose+debug boundaries are permitted.
|
||||
|
||||
``current_tier()`` is resolved once at bootstrap import; re-resolve it under
|
||||
a non-production env with ``CVDIAG_DEBUG=1`` so the §6 matrix lets
|
||||
``backend.sse.event`` (debug) and the verbose LLM boundaries through.
|
||||
"""
|
||||
import _shared.cvdiag_bootstrap as bootstrap
|
||||
|
||||
monkeypatch.setenv("SHOWCASE_ENV", "test")
|
||||
monkeypatch.setenv("CVDIAG_DEBUG", "1")
|
||||
bootstrap.setup({"SHOWCASE_ENV": "test", "CVDIAG_DEBUG": "1"})
|
||||
yield
|
||||
bootstrap.setup({"SHOWCASE_ENV": "test"})
|
||||
|
||||
|
||||
def _make_client(*, raise_server_exceptions: bool = True) -> TestClient:
|
||||
"""An app exposing three routes — happy stream, aborted stream, raise — each
|
||||
wrapped by the CVDIAG middleware. The endpoints emit the agent/LLM
|
||||
boundaries the middleware cannot observe, all keyed on the per-request ctx.
|
||||
"""
|
||||
|
||||
async def happy_stream(request):
|
||||
ctx = getattr(request.state, "cvdiag", None)
|
||||
if ctx is not None:
|
||||
emit_agent_enter(ctx, agent_name="showcase", model_id="gpt-4o-mini")
|
||||
|
||||
async def gen():
|
||||
if ctx is not None:
|
||||
async with LlmCallScope(
|
||||
ctx, provider="openai", model="gpt-4o-mini", interval_s=0.02
|
||||
):
|
||||
await asyncio.sleep(0.05) # let the heartbeat tick once
|
||||
yield b"data: hello\n\n"
|
||||
yield b"data: world\n\n"
|
||||
emit_agent_exit(ctx, terminal_outcome="ok", total_duration_ms=1)
|
||||
else:
|
||||
yield b"data: hello\n\n"
|
||||
|
||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||
|
||||
async def raises(request):
|
||||
raise RuntimeError("upstream rejected key sk-test-12345 Bearer abc.def.ghi")
|
||||
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route("/", happy_stream, methods=["POST"]),
|
||||
Route("/boom", raises, methods=["POST"]),
|
||||
]
|
||||
)
|
||||
app.add_middleware(CvdiagBackendMiddleware)
|
||||
return TestClient(app, raise_server_exceptions=raise_server_exceptions)
|
||||
|
||||
|
||||
async def _drive_abort() -> None:
|
||||
"""Drive the CVDIAG middleware over an unbounded stream and disconnect.
|
||||
|
||||
Builds the middleware around an unbounded inner stream, calls ``dispatch``
|
||||
to get the wrapped ``body_iterator``, reads one chunk, then ``aclose()``s it
|
||||
— the deterministic equivalent of a client disconnecting mid-stream. This
|
||||
raises ``GeneratorExit`` into the wrapper → ``backend.sse.aborted``.
|
||||
"""
|
||||
from starlette.requests import Request
|
||||
|
||||
async def unbounded():
|
||||
i = 0
|
||||
while True:
|
||||
yield f"data: chunk-{i}\n\n".encode()
|
||||
i += 1
|
||||
|
||||
inner_response = StreamingResponse(unbounded(), media_type="text/event-stream")
|
||||
|
||||
async def call_next(_request):
|
||||
return inner_response
|
||||
|
||||
scope = {
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/",
|
||||
"headers": [(b"x-aimock-context", b"ag2")],
|
||||
"query_string": b"",
|
||||
}
|
||||
|
||||
async def receive():
|
||||
return {"type": "http.request", "body": b""}
|
||||
|
||||
mw = CvdiagBackendMiddleware(app=lambda *a: None)
|
||||
request = Request(scope, receive)
|
||||
wrapped = await mw.dispatch(request, call_next)
|
||||
|
||||
body = wrapped.body_iterator
|
||||
await body.__anext__() # first chunk
|
||||
await body.aclose() # client disconnect mid-stream
|
||||
|
||||
|
||||
def test_all_eleven_backend_boundaries_emit(monkeypatch, capsys):
|
||||
"""All 11 backend boundaries emit across the three request shapes.
|
||||
|
||||
The happy stream yields ingress / agent.enter / llm.* / sse.first_byte /
|
||||
sse.event / agent.exit / response.complete; a disconnected stream yields
|
||||
sse.aborted; the raising route yields error.caught. Their union is the full
|
||||
eleven.
|
||||
"""
|
||||
monkeypatch.setenv("CVDIAG_BACKEND_EMITTER", "1")
|
||||
client = _make_client(raise_server_exceptions=False)
|
||||
|
||||
headers = {"x-test-id": VALID_TEST_ID, "x-aimock-context": "ag2"}
|
||||
resp = client.post("/", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Client-disconnect abort surface (→ backend.sse.aborted), driven directly
|
||||
# because Starlette's sync TestClient cannot reliably tear a stream down
|
||||
# mid-flight.
|
||||
asyncio.run(_drive_abort())
|
||||
|
||||
client.post("/boom", headers=headers)
|
||||
|
||||
envelopes = _parse_cvdiag_lines(capsys.readouterr().out)
|
||||
seen = _boundaries(envelopes)
|
||||
|
||||
missing = ALL_BACKEND_BOUNDARIES - seen
|
||||
assert not missing, (
|
||||
f"missing backend boundaries: {sorted(missing)}; saw {sorted(seen)}"
|
||||
)
|
||||
|
||||
# Correlation: every backend envelope carries the slug. The header-bearing
|
||||
# HTTP requests forward x-test-id verbatim; the directly driven abort
|
||||
# request mints its own UUIDv7 (no inbound header). Assert the forwarded
|
||||
# test_id appears on the header-bearing envelopes, and every minted id is a
|
||||
# well-formed UUIDv7.
|
||||
backend = [e for e in envelopes if e["layer"] == "backend"]
|
||||
assert backend, "no backend-layer envelopes emitted"
|
||||
assert all(e["slug"] == "ag2" for e in backend)
|
||||
forwarded = [e for e in backend if e["test_id"] == VALID_TEST_ID]
|
||||
assert forwarded, "forwarded x-test-id never appeared on any backend envelope"
|
||||
uuid7_re = __import__("re").compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
||||
)
|
||||
assert all(uuid7_re.match(e["test_id"]) for e in backend)
|
||||
# Closed 9-key edge-header bag always present on a header-bearing ingress.
|
||||
ingress = next(
|
||||
e
|
||||
for e in backend
|
||||
if e["boundary"] == "backend.request.ingress" and e["test_id"] == VALID_TEST_ID
|
||||
)
|
||||
assert set(ingress["edge_headers"].keys()) == {
|
||||
"cf-ray",
|
||||
"cf-mitigated",
|
||||
"cf-cache-status",
|
||||
"x-railway-edge",
|
||||
"x-railway-request-id",
|
||||
"x-hikari-trace",
|
||||
"retry-after",
|
||||
"via",
|
||||
"server",
|
||||
}
|
||||
|
||||
|
||||
def test_error_caught_scrubs_secret(monkeypatch, capsys):
|
||||
"""A synthetic ``sk-test-12345`` in an exception never reaches the emitted
|
||||
``backend.error.caught`` envelope."""
|
||||
monkeypatch.setenv("CVDIAG_BACKEND_EMITTER", "1")
|
||||
client = _make_client(raise_server_exceptions=False)
|
||||
|
||||
client.post("/boom", headers={"x-aimock-context": "ag2"})
|
||||
|
||||
out = capsys.readouterr().out
|
||||
envelopes = _parse_cvdiag_lines(out)
|
||||
errs = [e for e in envelopes if e["boundary"] == "backend.error.caught"]
|
||||
assert errs, "backend.error.caught not emitted"
|
||||
err = errs[0]
|
||||
assert err["metadata"]["exception_type"] == "RuntimeError"
|
||||
blob = json.dumps(err)
|
||||
assert "sk-test-12345" not in blob, "raw secret leaked into error envelope"
|
||||
assert "Bearer abc" not in blob, "raw bearer token leaked into error envelope"
|
||||
assert "[REDACTED]" in err["metadata"]["message_scrubbed"]
|
||||
|
||||
|
||||
def test_scrub_helper_redacts_known_secret_shapes():
|
||||
"""Unit-level: the scrub helper redacts bearer/sk-/pk-/userinfo shapes."""
|
||||
assert "sk-test-12345" not in scrub("key sk-test-12345 here")
|
||||
assert "sk-abcdefghijklmnopqrstuvwx" not in scrub("sk-abcdefghijklmnopqrstuvwx")
|
||||
assert "Bearer secrettoken" not in scrub("auth Bearer secrettoken")
|
||||
assert "pw" not in scrub("https://user:pw@host/path")
|
||||
assert scrub(None) == ""
|
||||
|
||||
|
||||
def test_heartbeat_fires_within_window(monkeypatch, capsys):
|
||||
"""``backend.llm.call.heartbeat`` fires while a slow LLM call is outstanding.
|
||||
|
||||
Uses a short interval so the test is fast; the production interval is ~10s
|
||||
and the spec requires a heartbeat within ~12s of a slow-LLM simulation —
|
||||
proven here by the same code path firing within its interval.
|
||||
"""
|
||||
monkeypatch.setenv("CVDIAG_BACKEND_EMITTER", "1")
|
||||
|
||||
async def run():
|
||||
ctx = _RequestCtx(test_id=VALID_TEST_ID, slug="ag2", demo="default")
|
||||
async with LlmCallScope(ctx, provider="openai", model="m", interval_s=0.05):
|
||||
await asyncio.sleep(0.18) # ~3 heartbeat intervals
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
envelopes = _parse_cvdiag_lines(capsys.readouterr().out)
|
||||
hb = [e for e in envelopes if e["boundary"] == "backend.llm.call.heartbeat"]
|
||||
assert hb, "no heartbeat emitted during a slow LLM call"
|
||||
assert all("elapsed_ms_since_start" in e["metadata"] for e in hb)
|
||||
|
||||
|
||||
def test_sse_aborted_on_client_disconnect(monkeypatch, capsys):
|
||||
"""Tearing the response stream down mid-flight emits ``backend.sse.aborted``
|
||||
with a ``termination_kind`` and the bytes streamed before the abort."""
|
||||
monkeypatch.setenv("CVDIAG_BACKEND_EMITTER", "1")
|
||||
|
||||
asyncio.run(_drive_abort())
|
||||
|
||||
envelopes = _parse_cvdiag_lines(capsys.readouterr().out)
|
||||
aborts = [e for e in envelopes if e["boundary"] == "backend.sse.aborted"]
|
||||
assert aborts, "backend.sse.aborted not emitted on client disconnect"
|
||||
meta = aborts[0]["metadata"]
|
||||
assert meta["termination_kind"] in {"rst", "timeout", "chunk_error"}
|
||||
assert meta["bytes_before_abort"] > 0
|
||||
# A disconnected stream must NOT also report a clean response.complete.
|
||||
completes = [e for e in envelopes if e["boundary"] == "backend.response.complete"]
|
||||
assert not completes, "clean response.complete emitted for an aborted stream"
|
||||
|
||||
|
||||
def test_disabled_by_default_emits_nothing(monkeypatch, capsys):
|
||||
"""With ``CVDIAG_BACKEND_EMITTER`` unset, NO backend CVDIAG line is emitted."""
|
||||
monkeypatch.delenv("CVDIAG_BACKEND_EMITTER", raising=False)
|
||||
client = _make_client()
|
||||
client.post("/", headers={"x-aimock-context": "ag2"})
|
||||
|
||||
envelopes = _parse_cvdiag_lines(capsys.readouterr().out)
|
||||
backend = [e for e in envelopes if e["layer"] == "backend"]
|
||||
assert backend == [], f"emitter fired while disabled: {backend}"
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Regression test: gen_ui_agent must import/construct without raising.
|
||||
|
||||
Reproduces the crash-on-import that took down `showcase-ag2` staging
|
||||
deploys since 76874f06d. Importing `agents.gen_ui_agent` constructs the
|
||||
`ConversableAgent` at module scope, which registers `set_steps` as an LLM
|
||||
tool. AG2 runs every tool parameter through pydantic
|
||||
`TypeAdapter(param).json_schema()`. When `set_steps` carries
|
||||
`from __future__ import annotations`, its `context_variables: ContextVariables`
|
||||
parameter is seen as an unresolved `ForwardRef('ContextVariables')`, and
|
||||
schema generation raises `pydantic.errors.PydanticUserError` at import time
|
||||
-> importing this agent module aborts server startup -> the service never
|
||||
comes up -> the Railway healthcheck (`/health`, owned by the integration's
|
||||
top-level entrypoint, not this module) fails.
|
||||
|
||||
The success criterion mirrors the failed healthcheck: the module imports
|
||||
cleanly and the agent object (with its registered tools) is built.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ConversableAgent construction validates that an LLM config / key is
|
||||
# present. The crash we are guarding against happens during tool-schema
|
||||
# generation, which is reached only after that validation, so seed a dummy
|
||||
# key. No network call is made at import time.
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key-not-used")
|
||||
|
||||
|
||||
def _gen_ui_agent_source_path() -> Path:
|
||||
"""Resolve src/agents/gen_ui_agent.py relative to this test file.
|
||||
|
||||
This test lives at <integration>/tests/python/, so the integration root
|
||||
is two levels up; the source then sits under src/agents/. Resolving via
|
||||
__file__ keeps the guard working regardless of where the repo is checked
|
||||
out (no hardcoded absolute path).
|
||||
"""
|
||||
integration_root = Path(__file__).resolve().parents[2]
|
||||
return integration_root / "src" / "agents" / "gen_ui_agent.py"
|
||||
|
||||
|
||||
def test_gen_ui_agent_source_has_no_future_annotations():
|
||||
"""The source must NOT carry `from __future__ import annotations`.
|
||||
|
||||
This is the load-bearing regression pin. `from __future__ import
|
||||
annotations` makes Python stringify every annotation (PEP 563), so the
|
||||
`set_steps` tool's `context_variables: ContextVariables` parameter is seen
|
||||
by AG2 as an unresolved `ForwardRef('ContextVariables')`. AG2 runs each
|
||||
tool parameter through pydantic `TypeAdapter(param).json_schema()`, which
|
||||
then raises `PydanticUserError` at import time -> importing this agent
|
||||
module aborts server startup -> the service never comes up -> the
|
||||
Railway/staging healthcheck (`/health`, owned by the entrypoint, not this
|
||||
module) fails (regression 76874f06d).
|
||||
|
||||
Asserting on the source text (not just "import didn't crash") makes this a
|
||||
TRUE guard: it fails the instant someone re-adds the future-import, even if
|
||||
a future AG2/pydantic happens to resolve the forward-ref gracefully and the
|
||||
import would no longer crash on its own.
|
||||
"""
|
||||
source_path = _gen_ui_agent_source_path()
|
||||
assert source_path.is_file(), f"could not locate source at {source_path}"
|
||||
|
||||
offending = "from __future__ import annotations"
|
||||
source_lines = source_path.read_text(encoding="utf-8").splitlines()
|
||||
matches = [
|
||||
f" line {i}: {line!r}"
|
||||
for i, line in enumerate(source_lines, start=1)
|
||||
if line.strip() == offending
|
||||
]
|
||||
assert not matches, (
|
||||
f"{source_path} must NOT contain `{offending}`.\n"
|
||||
"PEP 563 stringifies annotations, turning the set_steps tool's "
|
||||
"`context_variables: ContextVariables` into an unresolved ForwardRef; "
|
||||
"AG2's pydantic tool-schema generation then raises PydanticUserError "
|
||||
"at import time, so importing this agent module aborts server startup "
|
||||
"and the service never comes up, failing the Railway healthcheck "
|
||||
"(`/health`, owned by the entrypoint, not this module) (regression "
|
||||
"76874f06d). Remove the future-import.\nFound at:\n" + "\n".join(matches)
|
||||
)
|
||||
|
||||
|
||||
def test_gen_ui_agent_imports_without_pydantic_error():
|
||||
"""Importing the module must not raise PydanticUserError (or anything).
|
||||
|
||||
Proves the crash path is actually clear on the *installed* AG2/pydantic
|
||||
(complements the static source guard above, which is version-independent).
|
||||
"""
|
||||
try:
|
||||
module = importlib.import_module("agents.gen_ui_agent")
|
||||
except Exception: # noqa: BLE001 - re-raise so the verbatim traceback is kept
|
||||
# Let the original exception propagate with its full traceback intact:
|
||||
# for a deep pydantic schema-gen crash the verbatim stack IS the
|
||||
# load-bearing diagnostic, which a re-wrapped pytest.fail string loses.
|
||||
pytest.fail(
|
||||
"gen_ui_agent failed to import (see traceback below)",
|
||||
pytrace=True,
|
||||
)
|
||||
|
||||
# The agent and its ASGI app must be constructed (i.e. tool registration,
|
||||
# where the crash occurred, completed).
|
||||
assert module.agent is not None
|
||||
assert module.gen_ui_agent_app is not None
|
||||
# set_steps must be registered as a tool on the agent.
|
||||
tool_names = {t.name for t in module.agent.tools}
|
||||
assert "set_steps" in tool_names
|
||||
@@ -0,0 +1,433 @@
|
||||
"""Regression test: AG-UI image/document content parts must be rewritten
|
||||
to autogen-acceptable ``image_url`` parts before the multimodal sub-app
|
||||
hands the request to autogen's ``ConversableAgent``.
|
||||
|
||||
Failure under test
|
||||
==================
|
||||
The D6 ``multimodal`` probe sends a user message whose content list
|
||||
includes the modern AG-UI shape::
|
||||
|
||||
{"type": "image",
|
||||
"source": {"type": "data",
|
||||
"value": "<base64-png>",
|
||||
"mime_type": "image/png"}}
|
||||
|
||||
(plus a legacy ``{"type": "binary", "mimeType": ..., "data": ...}``
|
||||
mirror appended by ``src/app/demos/multimodal/legacy-converter-shim.tsx``
|
||||
to keep the @ag-ui/langgraph converter happy on other integrations).
|
||||
|
||||
Autogen's ``code_utils.content_str`` only accepts content-part types in
|
||||
``{"text", "input_text", "image_url", "input_image", "function",
|
||||
"tool_call", "tool_calls"}``. Anything else triggers::
|
||||
|
||||
ValueError("Wrong content format: unknown type <type> within the
|
||||
content")
|
||||
|
||||
…before the request reaches the vision model — observed live in the D6
|
||||
multimodal probe and recorded in commit d8a0a25db (which originally
|
||||
NSF-quarantined the feature).
|
||||
|
||||
The fix is ``NormalizingAGUIStream`` in ``agents._multimodal_normalize``,
|
||||
which subclasses ``AGUIStream`` and normalises the parsed
|
||||
``RunAgentInput`` messages AFTER Pydantic validation (where ``image`` is
|
||||
a valid AG-UI type) and BEFORE ``AgentService`` serialises them for
|
||||
autogen (where only ``image_url`` passes ``content_str``). The rewrite
|
||||
converts AG-UI image / document / binary parts to OpenAI Chat
|
||||
Completions ``image_url`` parts, leaving text and already-normalised
|
||||
parts untouched.
|
||||
|
||||
What this test asserts
|
||||
======================
|
||||
1. **RED → GREEN**: ``content_str`` raises on the raw AG-UI shape
|
||||
(``test_autogen_rejects_raw_agui_image_part``) but accepts the
|
||||
normalised output (``test_normalized_content_is_accepted_by_autogen``).
|
||||
This pins the fix to the actual autogen call site, not to a
|
||||
structural look-alike — if autogen ever relaxes the gate, the RED
|
||||
half of the pin will start passing and we'll know to revisit.
|
||||
2. **Shape coverage**: modern image/document data-source, modern
|
||||
url-source, legacy binary data/url, and text-passthrough cases
|
||||
each get a focused assertion.
|
||||
3. **Idempotency**: re-running the normalizer on already-normalised
|
||||
content is a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# autogen's ConversableAgent module-load path checks for an LLM key, even
|
||||
# though we never make a network call below — we only invoke the
|
||||
# allowed-types content gate. Seed a dummy value so import-time
|
||||
# validation passes regardless of the developer's shell env.
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key-not-used")
|
||||
|
||||
# Make ``agents._multimodal_normalize`` importable. The integration root
|
||||
# is two levels up (tests/python/ ⇒ <integration>/), the agents/ package
|
||||
# lives under src/.
|
||||
_INTEGRATION_ROOT = Path(__file__).resolve().parents[2]
|
||||
_SRC_ROOT = _INTEGRATION_ROOT / "src"
|
||||
if str(_SRC_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC_ROOT))
|
||||
|
||||
from agents._multimodal_normalize import ( # noqa: E402
|
||||
NormalizingAGUIStream,
|
||||
normalize_messages_for_autogen,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample payloads — small enough to read in-context, large enough to
|
||||
# exercise each AG-UI content shape the frontend actually emits.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# A 1x1 PNG, base64-encoded. Just enough bytes that data-URL assembly
|
||||
# is exercised; we never decode + render.
|
||||
_SAMPLE_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII="
|
||||
_SAMPLE_PDF_B64 = "JVBERi0xLjQKJYCAgIAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2c+PgplbmRvYmoK"
|
||||
|
||||
|
||||
def _modern_image_data_part() -> dict:
|
||||
return {
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "data",
|
||||
"value": _SAMPLE_PNG_B64,
|
||||
"mime_type": "image/png",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _modern_image_url_part() -> dict:
|
||||
return {
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "url",
|
||||
"value": "https://example.test/sample.png",
|
||||
"mime_type": "image/png",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _modern_document_data_part() -> dict:
|
||||
return {
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "data",
|
||||
"value": _SAMPLE_PDF_B64,
|
||||
"mime_type": "application/pdf",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _legacy_binary_data_part() -> dict:
|
||||
return {
|
||||
"type": "binary",
|
||||
"mimeType": "image/png",
|
||||
"data": _SAMPLE_PNG_B64,
|
||||
}
|
||||
|
||||
|
||||
def _legacy_binary_url_part() -> dict:
|
||||
return {
|
||||
"type": "binary",
|
||||
"mimeType": "image/png",
|
||||
"url": "https://example.test/sample.png",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RED/GREEN pin against autogen's actual content gate.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_autogen_rejects_raw_agui_image_part():
|
||||
"""Confirm the precise failure mode the normalizer is fixing.
|
||||
|
||||
Without normalization, autogen's ``content_str`` raises
|
||||
``ValueError`` with the verbatim message the D6 probe surfaced.
|
||||
This is the RED half of the pin: if autogen ever stops rejecting
|
||||
AG-UI image parts, this test starts failing and we'll know to
|
||||
revisit the normalizer (it may have become a no-op shim).
|
||||
"""
|
||||
pytest.importorskip("autogen")
|
||||
from autogen.code_utils import content_str
|
||||
|
||||
raw_content = [
|
||||
{"type": "text", "text": "describe the sample image"},
|
||||
_modern_image_data_part(),
|
||||
]
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
content_str(raw_content)
|
||||
assert "unknown type image" in str(exc_info.value), (
|
||||
"expected the exact ValueError text the D6 probe surfaced "
|
||||
"('Wrong content format: unknown type image within the "
|
||||
"content'); got: " + str(exc_info.value)
|
||||
)
|
||||
|
||||
|
||||
def test_normalized_content_is_accepted_by_autogen():
|
||||
"""The GREEN half of the pin: after normalization,
|
||||
``content_str`` accepts the user-message content list and returns
|
||||
a stringified placeholder for the image (autogen substitutes
|
||||
``<image>`` for any ``image_url`` part — see code_utils.py).
|
||||
"""
|
||||
pytest.importorskip("autogen")
|
||||
from autogen.code_utils import content_str
|
||||
|
||||
raw_messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe the sample image"},
|
||||
_modern_image_data_part(),
|
||||
],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(raw_messages)
|
||||
assert isinstance(normalised, list) and len(normalised) == 1
|
||||
user_content = normalised[0]["content"]
|
||||
# No exception expected — autogen's allowed-types gate accepts
|
||||
# every part in the rewritten list.
|
||||
rendered = content_str(user_content)
|
||||
assert "describe the sample image" in rendered
|
||||
# Autogen substitutes "<image>" for any image_url part. Asserting
|
||||
# on that substitution proves the part was recognised as an image
|
||||
# rather than skipped or rejected.
|
||||
assert "<image>" in rendered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shape-coverage assertions.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_modern_image_data_part_becomes_image_url_data_url():
|
||||
"""``{"type": "image", "source": {"type": "data", ...}}`` →
|
||||
``{"type": "image_url", "image_url": {"url": "data:<mime>;base64,<value>"}}``.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_modern_image_data_part()],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
part = normalised[0]["content"][0]
|
||||
assert part == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{_SAMPLE_PNG_B64}"},
|
||||
}
|
||||
|
||||
|
||||
def test_modern_image_url_part_keeps_remote_url():
|
||||
"""``{"type": "image", "source": {"type": "url", "value": "https://..."}}`` →
|
||||
``{"type": "image_url", "image_url": {"url": "https://..."}}``.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_modern_image_url_part()],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
assert normalised[0]["content"][0] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.test/sample.png"},
|
||||
}
|
||||
|
||||
|
||||
def test_modern_document_pdf_part_becomes_image_url_data_url():
|
||||
"""PDF documents survive the autogen allowed-types gate by
|
||||
riding inside an ``image_url`` data URL. The vision model still
|
||||
can't read the PDF directly, but at least the request reaches
|
||||
the model (which is the failure mode this fix targets — the
|
||||
upstream ``content_str`` ValueError before any model call).
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_modern_document_data_part()],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
part = normalised[0]["content"][0]
|
||||
assert part["type"] == "image_url"
|
||||
assert part["image_url"]["url"].startswith("data:application/pdf;base64,")
|
||||
assert part["image_url"]["url"].endswith(_SAMPLE_PDF_B64)
|
||||
|
||||
|
||||
def test_legacy_binary_data_part_becomes_image_url_data_url():
|
||||
"""``{"type": "binary", "mimeType": "image/png", "data": "..."}``
|
||||
(appended by legacy-converter-shim.tsx) is normalised the same way."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_legacy_binary_data_part()],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
assert normalised[0]["content"][0] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{_SAMPLE_PNG_B64}"},
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_binary_url_part_becomes_image_url_url():
|
||||
"""Legacy binary part with ``url`` field (no ``data``) keeps the
|
||||
URL intact as the image_url url."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [_legacy_binary_url_part()],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
assert normalised[0]["content"][0] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.test/sample.png"},
|
||||
}
|
||||
|
||||
|
||||
def test_text_only_user_message_passes_through_unchanged():
|
||||
"""Plain text content (the vast majority of turns) must hit the
|
||||
normalizer as a no-op — neither structurally rewritten nor
|
||||
re-wrapped — so non-multimodal demos never pay a behavioural cost
|
||||
from this fix."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "hello"}],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
# Identity preservation: when nothing changes, the same dict
|
||||
# objects are returned (not a deep copy). The middleware uses this
|
||||
# to skip body re-serialisation on no-op turns.
|
||||
assert normalised[0] is messages[0]
|
||||
assert normalised[0]["content"][0] == {"type": "text", "text": "hello"}
|
||||
|
||||
|
||||
def test_plain_string_content_passes_through_unchanged():
|
||||
"""User messages whose ``content`` is a plain string (the AG-UI
|
||||
text-only shape) are forwarded as-is."""
|
||||
messages = [
|
||||
{"role": "user", "content": "hello"},
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
assert normalised[0] is messages[0]
|
||||
|
||||
|
||||
def test_assistant_and_tool_messages_are_not_touched():
|
||||
"""Only user-role messages can carry AG-UI image content parts.
|
||||
Assistant / tool / system messages pass through unchanged."""
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": [_modern_image_data_part()]},
|
||||
{"role": "assistant", "content": "I see an image."},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "tool result",
|
||||
},
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
# Only the user message changed.
|
||||
assert normalised[0] is messages[0]
|
||||
assert normalised[1] is not messages[1]
|
||||
assert normalised[1]["content"][0]["type"] == "image_url"
|
||||
assert normalised[2] is messages[2]
|
||||
assert normalised[3] is messages[3]
|
||||
|
||||
|
||||
def test_normalize_is_idempotent():
|
||||
"""Running the normalizer on already-normalised content produces
|
||||
the same output, so a double-install of the middleware (mistake
|
||||
or otherwise) doesn't break the request."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe the sample image"},
|
||||
_modern_image_data_part(),
|
||||
],
|
||||
}
|
||||
]
|
||||
first = normalize_messages_for_autogen(messages)
|
||||
second = normalize_messages_for_autogen(first)
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_mimeType_alias_is_accepted():
|
||||
"""Some hand-rolled / older payloads use ``mimeType`` (camelCase)
|
||||
instead of the AG-UI pydantic ``mime_type``. The normaliser
|
||||
accepts both so a frontend running either schema version round-
|
||||
trips cleanly."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "data",
|
||||
"value": _SAMPLE_PNG_B64,
|
||||
"mimeType": "image/png",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
part = normalised[0]["content"][0]
|
||||
assert part["image_url"]["url"] == f"data:image/png;base64,{_SAMPLE_PNG_B64}"
|
||||
|
||||
|
||||
def test_unrecognised_image_source_drops_to_text_placeholder():
|
||||
"""If the modality is recognised (image/document/...) but the
|
||||
``source`` shape is malformed, the part is replaced by a text
|
||||
placeholder — autogen accepts the part and the user sees a
|
||||
triagable error rather than the request hard-failing with the
|
||||
autogen ValueError."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image", "source": {"type": "garbage"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
normalised = normalize_messages_for_autogen(messages)
|
||||
assert normalised[0]["content"][0] == {
|
||||
"type": "text",
|
||||
"text": "[unreadable image attachment]",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stream-class smoke: NormalizingAGUIStream is constructible and wraps
|
||||
# an agent correctly. We don't spin up uvicorn here — the unit-level
|
||||
# invariants above guard the regression; this is a tripwire that the
|
||||
# public surface stayed in place.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalizing_agui_stream_is_constructible():
|
||||
"""``NormalizingAGUIStream`` subclasses ``AGUIStream``, accepts a
|
||||
``ConversableAgent``, and exposes ``build_asgi()`` — the contract
|
||||
``multimodal_agent.py`` relies on."""
|
||||
from autogen import ConversableAgent, LLMConfig
|
||||
from autogen.ag_ui import AGUIStream
|
||||
|
||||
agent = ConversableAgent(
|
||||
name="test_agent",
|
||||
llm_config=LLMConfig({"model": "gpt-4o"}),
|
||||
human_input_mode="NEVER",
|
||||
)
|
||||
stream = NormalizingAGUIStream(agent)
|
||||
assert isinstance(stream, AGUIStream)
|
||||
assert callable(stream.build_asgi)
|
||||
Reference in New Issue
Block a user