chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+248
@@ -0,0 +1,248 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
// QA reference: qa/tool-rendering-reasoning-chain.md
|
||||
// Demo source: src/app/demos/tool-rendering-reasoning-chain/page.tsx
|
||||
//
|
||||
// The reasoning-chain cell composes two patterns into one chat surface:
|
||||
// - Reasoning-summary streaming (OpenAI Responses API, `reasoning={
|
||||
// "effort":"medium","summary":"detailed"}`) rendered through a
|
||||
// `messageView.reasoningMessage` slot (<ReasoningBlock>).
|
||||
// - Per-tool renderers wired via `useRenderTool` for `get_weather` and
|
||||
// `search_flights`, plus a `useDefaultRenderTool` catchall that
|
||||
// paints `get_stock_price` and `roll_dice`.
|
||||
//
|
||||
// Every pill drives a CHAINED two-tool flow:
|
||||
// - Stocks: get_stock_price(AAPL) → get_stock_price(MSFT) → comparison.
|
||||
// - Dice: roll_dice(sides=20) → roll_dice(sides=6) → contrast.
|
||||
// - Flights+weather: search_flights(SFO,JFK) → get_weather(JFK) → plan.
|
||||
//
|
||||
// Aimock fixtures live in showcase/aimock/d5-all.json (and the matching
|
||||
// harness source at showcase/harness/fixtures/d5/tool-rendering-
|
||||
// reasoning-chain.json) and pin every pill to a deterministic two-leg
|
||||
// chain. The sequential-pills test is the regression guard for the
|
||||
// AG-UI reasoning-role message bug in @copilotkit/runtime — without
|
||||
// `LangGraphAgent.run`'s reasoning-role filter, clicking a second pill
|
||||
// in the same thread used to crash with INCOMPLETE_STREAM because
|
||||
// @ag-ui/langgraph's message converter throws on `role:"reasoning"`.
|
||||
|
||||
const SUGGESTION_TIMEOUT = 15_000;
|
||||
const TOOL_TIMEOUT = 60_000;
|
||||
const REASONING_TIMEOUT = 30_000;
|
||||
|
||||
const PILLS = [
|
||||
"Compare two stocks",
|
||||
"Chain of dice rolls",
|
||||
"Flights + destination weather",
|
||||
] as const;
|
||||
|
||||
test.describe("Tool Rendering — Reasoning Chain", () => {
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto("/demos/tool-rendering-reasoning-chain");
|
||||
await expect(page.getByPlaceholder("Type a message")).toBeVisible({
|
||||
timeout: SUGGESTION_TIMEOUT,
|
||||
});
|
||||
});
|
||||
|
||||
test("page loads with composer and 3 suggestion pills", async ({ page }) => {
|
||||
const suggestions = page.locator('[data-testid="copilot-suggestion"]');
|
||||
for (const title of PILLS) {
|
||||
await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({
|
||||
timeout: SUGGESTION_TIMEOUT,
|
||||
});
|
||||
}
|
||||
|
||||
// Sanity: no per-tool cards mounted before any pill click.
|
||||
await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0);
|
||||
await expect(page.locator('[data-testid="flight-list-card"]')).toHaveCount(
|
||||
0,
|
||||
);
|
||||
await expect(
|
||||
page.locator('[data-testid="custom-catchall-card"]'),
|
||||
).toHaveCount(0);
|
||||
await expect(page.locator('[data-testid="reasoning-block"]')).toHaveCount(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test("Compare two stocks pill chains AAPL → MSFT through the catchall renderer", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Compare two stocks" })
|
||||
.first()
|
||||
.click();
|
||||
|
||||
// Both legs of the chain mount via the catchall renderer — scoped
|
||||
// by `data-tool-name` so we'd notice if a future per-tool stock
|
||||
// renderer landed and only one card rendered.
|
||||
const stockCards = page.locator(
|
||||
'[data-testid="custom-catchall-card"][data-tool-name="get_stock_price"]',
|
||||
);
|
||||
await expect
|
||||
.poll(async () => stockCards.count(), { timeout: TOOL_TIMEOUT })
|
||||
.toBe(2);
|
||||
|
||||
// Reasoning slot mounts at least once — proves the agent's
|
||||
// reasoning summaries reached the messageView slot, which is the
|
||||
// whole reason this cell exists vs the plain tool-rendering demo.
|
||||
await expect(
|
||||
page.locator('[data-testid="reasoning-block"]').first(),
|
||||
).toBeVisible({ timeout: REASONING_TIMEOUT });
|
||||
|
||||
// Narration text comes from the fixture final-content leg.
|
||||
await expect(page.getByText("AAPL is at")).toBeVisible({
|
||||
timeout: TOOL_TIMEOUT,
|
||||
});
|
||||
await expect(page.getByText("MSFT is at")).toBeVisible({
|
||||
timeout: TOOL_TIMEOUT,
|
||||
});
|
||||
});
|
||||
|
||||
test("Chain of dice rolls pill chains d20 → d6 through the catchall renderer", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Chain of dice rolls" })
|
||||
.first()
|
||||
.click();
|
||||
|
||||
const diceCards = page.locator(
|
||||
'[data-testid="custom-catchall-card"][data-tool-name="roll_dice"]',
|
||||
);
|
||||
await expect
|
||||
.poll(async () => diceCards.count(), { timeout: TOOL_TIMEOUT })
|
||||
.toBe(2);
|
||||
|
||||
await expect(
|
||||
page.locator('[data-testid="reasoning-block"]').first(),
|
||||
).toBeVisible({ timeout: REASONING_TIMEOUT });
|
||||
|
||||
// Final narration mentions both dice + the contrast framing.
|
||||
await expect(page.getByText(/d20 came up/i)).toBeVisible({
|
||||
timeout: TOOL_TIMEOUT,
|
||||
});
|
||||
});
|
||||
|
||||
test("Flights + destination weather pill chains search_flights → get_weather through branded per-tool renderers", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Flights + destination weather" })
|
||||
.first()
|
||||
.click();
|
||||
|
||||
// Flights card uses its branded renderer (not the catchall).
|
||||
const flights = page.locator('[data-testid="flight-list-card"]').first();
|
||||
await expect(flights).toBeVisible({ timeout: TOOL_TIMEOUT });
|
||||
await expect(
|
||||
flights.locator('[data-testid="flight-origin"]'),
|
||||
).toContainText("SFO", { timeout: TOOL_TIMEOUT });
|
||||
await expect(
|
||||
flights.locator('[data-testid="flight-destination"]'),
|
||||
).toContainText("JFK", { timeout: TOOL_TIMEOUT });
|
||||
|
||||
// Destination weather card uses its branded renderer.
|
||||
const weather = page.locator('[data-testid="weather-card"]').first();
|
||||
await expect(weather).toBeVisible({ timeout: TOOL_TIMEOUT });
|
||||
await expect(weather.locator('[data-testid="weather-city"]')).toContainText(
|
||||
"JFK",
|
||||
{ timeout: TOOL_TIMEOUT },
|
||||
);
|
||||
|
||||
await expect(
|
||||
page.locator('[data-testid="reasoning-block"]').first(),
|
||||
).toBeVisible({ timeout: REASONING_TIMEOUT });
|
||||
|
||||
// Catchall renderer must NOT mount for these tools — both have
|
||||
// per-tool registrations.
|
||||
await expect(
|
||||
page.locator('[data-testid="custom-catchall-card"]'),
|
||||
).toHaveCount(0);
|
||||
});
|
||||
|
||||
// REGRESSION for the AG-UI reasoning-role message bug:
|
||||
// `@ag-ui/langgraph`'s message converter throws "message role is
|
||||
// not supported." on any role outside {user,assistant,system,tool}.
|
||||
// Reasoning-stream agents emit `role:"reasoning"` messages that the
|
||||
// AG-UI client replays on subsequent turns. Without the
|
||||
// reasoning-role filter in @copilotkit/runtime's LangGraphAgent.run
|
||||
// subclass, the SECOND pill click crashes before the model is
|
||||
// called and the user sees a runtime error toast.
|
||||
//
|
||||
// This test clicks all three pills sequentially in ONE thread and
|
||||
// asserts the full chain renders for each — proving cross-turn safety.
|
||||
// It also catches a regression in any of:
|
||||
// - The fixture toolCallId chains (degrading multi-pill to single
|
||||
// tool calls).
|
||||
// - The reasoning summary emission on follow-up turns.
|
||||
// - Per-tool renderer state isolation between turns.
|
||||
test("sequential pills in one thread render full chains + reasoning blocks for each", async ({
|
||||
page,
|
||||
}) => {
|
||||
// Three sequential pills × 2-tool chains × LLM-mock latency easily
|
||||
// exceeds Playwright's 30s default. Match the budget the
|
||||
// tool-rendering-default-catchall multi-pill regression uses.
|
||||
test.setTimeout(240_000);
|
||||
|
||||
const reasoningBlocks = page.locator('[data-testid="reasoning-block"]');
|
||||
|
||||
// Pill 1 — stocks chain.
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Compare two stocks" })
|
||||
.first()
|
||||
.click();
|
||||
const stockCards = page.locator(
|
||||
'[data-testid="custom-catchall-card"][data-tool-name="get_stock_price"]',
|
||||
);
|
||||
await expect
|
||||
.poll(async () => stockCards.count(), { timeout: TOOL_TIMEOUT })
|
||||
.toBe(2);
|
||||
await expect
|
||||
.poll(async () => reasoningBlocks.count(), { timeout: REASONING_TIMEOUT })
|
||||
.toBeGreaterThanOrEqual(1);
|
||||
|
||||
// Pill 2 — dice chain. The KEY assertion: this used to crash with
|
||||
// INCOMPLETE_STREAM before the reasoning-role filter landed.
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Chain of dice rolls" })
|
||||
.first()
|
||||
.click();
|
||||
const diceCards = page.locator(
|
||||
'[data-testid="custom-catchall-card"][data-tool-name="roll_dice"]',
|
||||
);
|
||||
await expect
|
||||
.poll(async () => diceCards.count(), { timeout: TOOL_TIMEOUT })
|
||||
.toBe(2);
|
||||
// Reasoning blocks should have INCREASED — proves the second turn
|
||||
// produced fresh reasoning, not just reusing turn 1's block.
|
||||
await expect
|
||||
.poll(async () => reasoningBlocks.count(), { timeout: REASONING_TIMEOUT })
|
||||
.toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Pill 3 — flights + destination weather. Final regression hop.
|
||||
await page
|
||||
.locator('[data-testid="copilot-suggestion"]')
|
||||
.filter({ hasText: "Flights + destination weather" })
|
||||
.first()
|
||||
.click();
|
||||
await expect(
|
||||
page.locator('[data-testid="flight-list-card"]').first(),
|
||||
).toBeVisible({ timeout: TOOL_TIMEOUT });
|
||||
await expect(
|
||||
page.locator('[data-testid="weather-card"]').first(),
|
||||
).toBeVisible({ timeout: TOOL_TIMEOUT });
|
||||
await expect
|
||||
.poll(async () => reasoningBlocks.count(), { timeout: REASONING_TIMEOUT })
|
||||
.toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Final sanity: card counts for the prior turns survived (no
|
||||
// unmounts mid-thread).
|
||||
await expect(stockCards).toHaveCount(2);
|
||||
await expect(diceCards).toHaveCount(2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user