/** * E2E Smoke Test Suite for Showcase Integrations * * Tests the full path: Railway backend -> AG-UI protocol -> CopilotKit runtime -> frontend response * * Levels: * 1. @health - Backend health endpoint returns 200 * 2. @agent - Agent endpoint is reachable (not 404) * 3. @chat - Round-trip chat: send message, get assistant response * 4. @tools - Tool rendering: trigger a tool, verify UI result * * Run all: npx playwright test * Run level: npx playwright test --grep @health * Run one: npx playwright test --grep "langgraph-python" */ import { test, expect } from "@playwright/test"; import { checkHealth, checkAgentEndpoint, sendChatMessage } from "./helpers"; // Generated by showcase/scripts/generate-registry.ts — must run `npm run build` // in showcase/shell (or run the generator directly) before this test can import the file. import registry from "../../shell/src/data/registry.json"; import localPorts from "../../shared/local-ports.json"; // LOCAL_PORTS=1 rewrites Railway backend URLs to http://localhost: // using showcase/shared/local-ports.json. Lets smoke run against the // docker-compose.local.yml stack instead of Railway. Starters are skipped // because they're not represented in local-ports.json (local dev only // targets the 17 integration backends). const USE_LOCAL_PORTS = process.env.LOCAL_PORTS === "1"; // SHOWCASE_BACKEND_HOST_PATTERN lets a single deployed test image be // re-pointed at a different backend environment (e.g. staging) without // regenerating registry.json. `{slug}` is the only placeholder. If unset, // each integration's baked-in backend_url is used as-is (current behavior). // LOCAL_PORTS=1 takes precedence so docker-compose flows are unaffected. const HOST_PATTERN_OVERRIDE = process.env.SHOWCASE_BACKEND_HOST_PATTERN || ""; const applyHostPattern = (slug: string, defaultUrl: string): string => { if (!HOST_PATTERN_OVERRIDE) return defaultUrl; return `https://${HOST_PATTERN_OVERRIDE.replace("{slug}", slug)}`; }; const rewriteBackendUrl = (slug: string, railwayUrl: string): string => { if (USE_LOCAL_PORTS) { const port = (localPorts as Record)[slug]; if (!port) throw new Error(`LOCAL_PORTS=1 but no port for slug '${slug}'`); return `http://localhost:${port}`; } return applyHostPattern(slug, railwayUrl); }; // --------------------------------------------------------------------------- // Integration registry — derived from showcase/shell/src/data/registry.json // --------------------------------------------------------------------------- const INTEGRATIONS = registry.integrations.map((i) => ({ slug: i.slug, name: i.name, backendUrl: i.backend_url, deployed: i.deployed, hasToolRendering: i.features.includes("tool-rendering"), demos: i.demos.map((d: { id: string }) => d.id), })); // Only test deployed integrations unless SMOKE_ALL=true const SMOKE_ALL = process.env.SMOKE_ALL === "true"; const activeIntegrations = ( SMOKE_ALL ? INTEGRATIONS : INTEGRATIONS.filter((i) => i.deployed) ).map((i) => ({ ...i, backendUrl: rewriteBackendUrl(i.slug, i.backendUrl) })); // --------------------------------------------------------------------------- // Level 1: Health checks (@health) — fast, API-only // --------------------------------------------------------------------------- test.describe("Level 1: Backend Health @health", () => { for (const integration of activeIntegrations) { test(`[L1: health] ${integration.slug} backend is healthy @health`, async ({ request, }) => { const result = await checkHealth(request, integration.backendUrl); expect( result.ok, `${integration.slug} health check failed: status=${result.status} path=${result.path} body=${result.body.slice(0, 500)}`, ).toBe(true); }); } }); // --------------------------------------------------------------------------- // Level 2: Agent endpoint reachability (@agent) // --------------------------------------------------------------------------- test.describe("Level 2: Agent Endpoint @agent", () => { for (const integration of activeIntegrations) { test(`[L2: agent] ${integration.slug} agent endpoint responds (not 404) @agent`, async ({ request, }) => { const result = await checkAgentEndpoint(request, integration.backendUrl); expect( result.status, `${integration.slug} agent endpoint returned 404 — likely using wrong agent type (LangGraphAgent vs HttpAgent). Status=${result.status} body=${result.body.slice(0, 500)}`, ).not.toBe(404); expect( result.ok, `${integration.slug} agent endpoint is unreachable: status=${result.status} body=${result.body.slice(0, 500)}`, ).toBe(true); }); } }); // --------------------------------------------------------------------------- // Level 3: Round-trip chat (@chat) — browser-based // --------------------------------------------------------------------------- test.describe("Level 3: Round-trip Chat @chat", () => { for (const integration of activeIntegrations) { test(`[L3: chat] ${integration.slug} responds to a message @chat`, async ({ page, }) => { test.slow(); // Allow extra time for cold starts const result = await sendChatMessage( page, integration.backendUrl, "Hello, please respond with a brief greeting.", "/demos/agentic-chat", ); expect( result.gotResponse, `${integration.slug} did not produce an assistant response. The agent may be down, misconfigured, or using the wrong agent type.`, ).toBe(true); expect( result.responseText.length, `${integration.slug} assistant response was empty`, ).toBeGreaterThan(0); }); } }); // --------------------------------------------------------------------------- // Level 4: Tool rendering (@tools) — browser-based, subset // --------------------------------------------------------------------------- test.describe("Level 4: Tool Rendering @tools", () => { // Future-proofing: `demos.includes("tool-rendering")` is what actually matters — // if a new integration lands without the tool-rendering demo, it's correctly // skipped here. The previous `hasToolRendering` boolean was redundant with // this check (always `true` alongside the demo entry) and has been removed. const toolIntegrations = activeIntegrations.filter((i) => i.demos.includes("tool-rendering"), ); for (const integration of toolIntegrations) { test(`[L4: tools] ${integration.slug} renders tool results @tools`, async ({ page, }) => { test.slow(); const result = await sendChatMessage( page, integration.backendUrl, "What's the weather in San Francisco?", "/demos/tool-rendering", ); // Tool rendering should produce an assistant response expect( result.gotResponse, `${integration.slug} did not render a tool result`, ).toBe(true); // The response should contain weather-related content. Avoid a bare // `/\d+/` fallback — any digit anywhere makes the assertion vacuous // (timestamps, UI chrome, model names all contain digits). Match only // on weather-specific vocabulary. const responseLC = result.responseText.toLowerCase(); const hasWeatherContent = responseLC.includes("san francisco") || responseLC.includes("weather") || responseLC.includes("temperature") || responseLC.includes("degrees") || responseLC.includes("sunny") || responseLC.includes("cloudy") || responseLC.includes("rain") || responseLC.includes("humidity") || responseLC.includes("wind"); expect( hasWeatherContent, `${integration.slug} response doesn't contain weather info: "${result.responseText}"`, ).toBe(true); // Check for rendered components (not just text) — catches the query_data // loop bug where the agent returns text but never renders a chart/card. // // isVisible() resolves false for missing/detached elements rather than // throwing, so the previous .catch(() => false) was masking nothing on // the happy path — but it ALSO masked real errors (e.g. evaluation // errors in the locator) as "not visible", which is the concrete bug // the original TODO in this file flagged. Drop the swallowing catches // and let genuine errors surface as test failures; a missing optional // renderer still just yields `false`. const responseArea = page .locator('[data-testid="copilot-assistant-message"]') .last(); const checks = await Promise.all([ responseArea.locator(".recharts-wrapper").first().isVisible(), responseArea .locator("svg circle[stroke-dasharray]") .first() .isVisible(), responseArea .locator('[data-testid="weather-card"]') .first() .isVisible(), responseArea.locator("canvas").first().isVisible(), ]); const hasRenderedComponent = checks.some(Boolean); // Warning for now — upgrade to hard failure after data-testid convention // is established across all integrations if (!hasRenderedComponent) { console.warn( `\u26a0\ufe0f ${integration.slug}: Tool response contains text but no rendered chart/weather component. This would have missed the query_data loop bug.`, ); } }); } });