import { test, expect } from "@playwright/test"; // QA reference: qa/tool-rendering-custom-catchall.md // Demo source: src/app/demos/tool-rendering-custom-catchall/page.tsx // Renderer source: src/app/demos/tool-rendering-custom-catchall/custom-catchall-renderer.tsx // // This cell registers a SINGLE branded wildcard renderer via // `useDefaultRenderTool`. Every tool call must paint via the same // `[data-testid="custom-wildcard-card"]` shell — no per-tool // specialization. Test 6 is the load-bearing assertion: every card on // the page after each pill click shares the same testid signature. const SUGGESTION_TIMEOUT = 15000; const TOOL_TIMEOUT = 60000; const PILLS = ["Weather in SF", "Find flights", "Roll a d20", "Chain tools"]; test.describe("Tool Rendering — Custom Catch-all (branded wildcard)", () => { test.beforeEach(async ({ page }) => { await page.goto("/demos/tool-rendering-custom-catchall"); await expect(page.getByPlaceholder("Type a message")).toBeVisible({ timeout: SUGGESTION_TIMEOUT, }); }); test("page loads with composer and 4 suggestion pills", async ({ page }) => { const suggestions = page.locator('[data-testid="copilot-suggestion"]'); for (const title of PILLS) { await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({ timeout: SUGGESTION_TIMEOUT, }); } // Sanity: per-tool branded testids from sibling cells stay at zero. await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="stock-card"]')).toHaveCount(0); await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0); // Sanity: the OOTB default-renderer testid does NOT appear here. await expect( page.locator('[data-testid="copilot-tool-render"]'), ).toHaveCount(0); }); test("Weather in SF pill paints the branded wildcard card for get_weather", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Weather in SF" }) .first() .click(); const card = page .locator( '[data-testid="custom-wildcard-card"][data-tool-name="get_weather"]', ) .first(); await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( card.locator('[data-testid="custom-wildcard-tool-name"]'), ).toHaveText("get_weather"); await expect( card.locator('[data-testid="custom-wildcard-args"]'), ).toContainText("San Francisco", { timeout: TOOL_TIMEOUT }); }); test("Find flights pill paints the SAME branded wildcard card for search_flights", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Find flights" }) .first() .click(); const card = page .locator( '[data-testid="custom-wildcard-card"][data-tool-name="search_flights"]', ) .first(); await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( card.locator('[data-testid="custom-wildcard-tool-name"]'), ).toHaveText("search_flights"); // Result block surfaces the deterministic flights from our fixture // (NOT the a2ui beautiful-chat boilerplate). await expect( card.locator('[data-testid="custom-wildcard-result"]'), ).toContainText(/United|Delta|JetBlue/, { timeout: TOOL_TIMEOUT }); }); test("Roll a d20 pill paints exactly 5 wildcard cards, last result is 20", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Roll a d20" }) .first() .click(); const cards = page.locator( '[data-testid="custom-wildcard-card"][data-tool-name="roll_d20"]', ); await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBe(5); // 5th card's result is 20. await expect( cards.nth(4).locator('[data-testid="custom-wildcard-result"]'), ).toContainText(/"value":\s*20|"result":\s*20/, { timeout: TOOL_TIMEOUT }); // First 4 are non-20. for (let i = 0; i < 4; i++) { const txt = await cards .nth(i) .locator('[data-testid="custom-wildcard-result"]') .innerText(); expect(txt).not.toMatch(/"value":\s*20|"result":\s*20/); } }); test("Chain tools pill paints 3 wildcard cards (one per tool)", async ({ page, }) => { await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Chain tools" }) .first() .click(); await expect( page .locator( '[data-testid="custom-wildcard-card"][data-tool-name="get_weather"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( page .locator( '[data-testid="custom-wildcard-card"][data-tool-name="search_flights"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); await expect( page .locator( '[data-testid="custom-wildcard-card"][data-tool-name="roll_d20"]', ) .first(), ).toBeVisible({ timeout: TOOL_TIMEOUT }); }); test("every rendered card shares the same wildcard testid signature", async ({ page, }) => { // Cross-tool sanity: drive Chain tools (3 distinct tools → 3 // cards) and assert every card matches the same wildcard shell. await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Chain tools" }) .first() .click(); const cards = page.locator('[data-testid="custom-wildcard-card"]'); await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBeGreaterThanOrEqual(3); const total = await cards.count(); await expect( page.locator('[data-testid="custom-wildcard-tool-name"]'), ).toHaveCount(total); await expect( page.locator('[data-testid="custom-wildcard-args"]'), ).toHaveCount(total); // All cards expose distinct tool names but the SAME shell. const toolNames = await cards.evaluateAll((nodes) => nodes.map((n) => n.getAttribute("data-tool-name")), ); const uniqueNames = new Set(toolNames); expect(uniqueNames.size).toBeGreaterThanOrEqual(3); for (const name of toolNames) { expect(["get_weather", "search_flights", "roll_d20"]).toContain(name); } // The OOTB default-renderer testid stays at zero — proves the // single custom wildcard is what painted, not the framework // fallback. await expect( page.locator('[data-testid="copilot-tool-render"]'), ).toHaveCount(0); }); // Regression for the aimock multi-pill bug: // The d20 and Chain-tools fixtures used global thread state // (`turnIndex`, `hasToolResult`) to drive sequencing. After clicking // Find flights, the d20 loop entered at turnIndex=2 (rendering only 3 // cards instead of 5) and the Chain-tools tool-emitting fixture was // skipped entirely (no tool cards, just the final "Done — Tokyo is // sunny…" content). Fix: chain all follow-ups via `toolCallId`. This // test drives Find flights → Roll a d20 → Chain tools in one thread // and asserts the wildcard renderer paints the full card sequence for // every pill (1 + 5 + 3 = 9 cards). test("sequential pills in one thread render full card sequences for each", async ({ page, }) => { // Three sequential pills × multi-tool chains × LLM-mock latency easily // exceeds Playwright's 30s default. Bumped to cover the worst case. test.setTimeout(240_000); await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Find flights" }) .first() .click(); const cards = page.locator('[data-testid="custom-wildcard-card"]'); await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBe(1); await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Roll a d20" }) .first() .click(); // 1 (flights) + 5 (d20) = 6 cards once d20 chain finishes. await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBe(6); await expect(page.getByText("Rolled the d20 five times")).toBeVisible({ timeout: TOOL_TIMEOUT, }); await page .locator('[data-testid="copilot-suggestion"]') .filter({ hasText: "Chain tools" }) .first() .click(); // 1 + 5 + 3 = 9 once chain tools mounts get_weather + search_flights + // roll_d20 cards. await expect .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT }) .toBe(9); await expect(page.getByText("Done — Tokyo is sunny")).toBeVisible({ timeout: TOOL_TIMEOUT, }); }); });