import { test, expect } from "@playwright/test"; // QA reference: qa/tool-rendering-default-catchall.md // Demo source: src/app/demos/tool-rendering-default-catchall/page.tsx // // This cell registers ZERO custom render hooks. The runtime falls back // to the framework's built-in DefaultToolCallRenderer, which paints // every tool call with a stable `[data-testid="copilot-tool-render"]` // wrapper plus a `data-tool-name=""` attribute. We assert on the // built-in contract — branded testids from sibling cells stay at zero. const SUGGESTION_TIMEOUT = 15000; const TOOL_TIMEOUT = 60000; const PILLS = ["Weather in SF", "Find flights", "Roll a d20", "Chain tools"]; test.describe("Tool Rendering — Default Catch-all", () => { test.beforeEach(async ({ page }) => { await page.goto("/demos/tool-rendering-default-catchall"); await expect(page.getByPlaceholder("Type a message")).toBeVisible({ timeout: SUGGESTION_TIMEOUT, }); }); test("page loads with composer and 4 suggestion pills", async ({ page }) => { const suggestions = page.locator('[data-testid="copilot-suggestion"]'); for (const title of PILLS) { await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({ timeout: SUGGESTION_TIMEOUT, }); } // Sanity: branded sibling-cell testids stay at zero on this cell. await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="stock-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0); await expect( page.locator('[data-testid="custom-wildcard-card"]'), ).toHaveCount(0); }); test("Weather in SF pill paints the built-in default card for get_weather", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Weather in SF" }) .first() .click(); const card = page .locator( '[data-testid="copilot-tool-render"][data-tool-name="get_weather"]', ) .first(); await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT }); // Args are pinned to San Francisco (verbatim pill prompt → fixture). await expect .poll(async () => card.getAttribute("data-args"), { timeout: TOOL_TIMEOUT, }) .toContain("San Francisco"); // No branded sibling-cell card mounted. await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0); await expect( page.locator('[data-testid="custom-wildcard-card"]'), ).toHaveCount(0); }); test("Find flights pill paints the built-in default card for search_flights", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Find flights" }) .first() .click(); const card = page .locator( '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]', ) .first(); await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT }); // Result attribute carries the deterministic fixture flights (NOT // the a2ui beautiful-chat shape). await expect .poll(async () => card.getAttribute("data-result"), { timeout: TOOL_TIMEOUT, }) .toMatch(/United|Delta|JetBlue/); await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0); }); test("Roll a d20 pill paints exactly 5 default cards for roll_d20", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Roll a d20" }) .first() .click(); const cards = page.locator( '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]', ); await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBe(5); // 5th card's result must contain "20" (the final scripted roll). const lastResult = await cards.nth(4).getAttribute("data-result"); expect(lastResult ?? "").toMatch(/"value":\s*20|"result":\s*20/); // First 4 results are not-20. for (let i = 0; i < 4; i++) { const r = (await cards.nth(i).getAttribute("data-result")) ?? ""; expect(r).not.toMatch(/"value":\s*20|"result":\s*20/); } await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0); }); test("Chain tools pill paints 3 default cards (weather + flights + d20)", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Chain tools" }) .first() .click(); await expect( page .locator( '[data-testid="copilot-tool-render"][data-tool-name="get_weather"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( page .locator( '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( page .locator( '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); }); // Regression for the aimock multi-pill bug: // The d20 and Chain-tools fixtures used `turnIndex` + `hasToolResult` to // disambiguate sequential iterations of the same prompt. Those gates // count *global* thread state: clicking Find flights first left two // assistant messages and one tool message behind, so the d20 loop // entered at `turnIndex=2` (skipping rolls 7 and 14, hence only 3 // cards), and the Chain-tools tool-emitting fixture was skipped // entirely (`hasToolResult: false` failed) so the pill went straight to // the "Done — Tokyo is sunny…" content with no tool cards. Fix: chain // all follow-ups via `toolCallId`, drop the global gates. This test // drives the three offending pills in a single thread and asserts the // expected card counts for each. test("sequential pills in one thread render full card sequences for each", async ({ page, }) => { // Three sequential pills × multi-tool chains × LLM-mock latency easily // exceeds Playwright's 30s default. Bumped to cover the worst case. test.setTimeout(240_000); await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Find flights" }) .first() .click(); const flights = page.locator( '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]', ); await expect(flights).toHaveCount(1, { timeout: TOOL_TIMEOUT }); await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Roll a d20" }) .first() .click(); const d20 = page.locator( '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]', ); await expect .poll(async () => d20.count(), { timeout: TOOL_TIMEOUT }) .toBe(5); // Final scripted roll lands the 20 — proves the chain advanced through // all 5 fixtures, not just the first two before bailing to content. await expect .poll(async () => d20.nth(4).getAttribute("data-result"), { timeout: TOOL_TIMEOUT, }) .toMatch(/"value":\s*20|"result":\s*20/); await expect(page.getByText("Rolled the d20 five times")).toBeVisible({ timeout: TOOL_TIMEOUT, }); }); test("every rendered card matches the built-in default-renderer DOM signature", async ({ page, }) => { // Drive a single pill that produces a single card so the assertions // here are scoped to the exact DOM the framework's default renderer // produces. await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Weather in SF" }) .first() .click(); const card = page.locator('[data-testid="copilot-tool-render"]').first(); await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT }); // The built-in default renderer always exposes name + status pill. await expect( card.locator('[data-testid="copilot-tool-render-name"]'), ).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( card.locator('[data-testid="copilot-tool-render-status"]'), ).toBeVisible({ timeout: TOOL_TIMEOUT }); // Every card on the page shares the same wrapper testid count as // the inner-name and inner-status testids — proves the built-in // shell is what's painting (no per-tool shells). const total = await page .locator('[data-testid="copilot-tool-render"]') .count(); await expect( page.locator('[data-testid="copilot-tool-render-name"]'), ).toHaveCount(total); await expect( page.locator('[data-testid="copilot-tool-render-status"]'), ).toHaveCount(total); }); });