wehub-resource-sync
230 lines
9.4 KiB
TypeScript
230 lines
9.4 KiB
TypeScript
/**
|
|
* E2E Smoke Test Suite for Showcase Integrations
|
|
*
|
|
* Tests the full path: Railway backend -> AG-UI protocol -> CopilotKit runtime -> frontend response
|
|
*
|
|
* Levels:
|
|
* 1. @health - Backend health endpoint returns 200
|
|
* 2. @agent - Agent endpoint is reachable (not 404)
|
|
* 3. @chat - Round-trip chat: send message, get assistant response
|
|
* 4. @tools - Tool rendering: trigger a tool, verify UI result
|
|
*
|
|
* Run all: npx playwright test
|
|
* Run level: npx playwright test --grep @health
|
|
* Run one: npx playwright test --grep "langgraph-python"
|
|
*/
|
|
|
|
import { test, expect } from "@playwright/test";
|
|
import { checkHealth, checkAgentEndpoint, sendChatMessage } from "./helpers";
|
|
// Generated by showcase/scripts/generate-registry.ts — must run `npm run build`
|
|
// in showcase/shell (or run the generator directly) before this test can import the file.
|
|
import registry from "../../shell/src/data/registry.json";
|
|
import localPorts from "../../shared/local-ports.json";
|
|
|
|
// LOCAL_PORTS=1 rewrites Railway backend URLs to http://localhost:<port>
|
|
// using showcase/shared/local-ports.json. Lets smoke run against the
|
|
// docker-compose.local.yml stack instead of Railway. Starters are skipped
|
|
// because they're not represented in local-ports.json (local dev only
|
|
// targets the 17 integration backends).
|
|
const USE_LOCAL_PORTS = process.env.LOCAL_PORTS === "1";
|
|
|
|
// SHOWCASE_BACKEND_HOST_PATTERN lets a single deployed test image be
|
|
// re-pointed at a different backend environment (e.g. staging) without
|
|
// regenerating registry.json. `{slug}` is the only placeholder. If unset,
|
|
// each integration's baked-in backend_url is used as-is (current behavior).
|
|
// LOCAL_PORTS=1 takes precedence so docker-compose flows are unaffected.
|
|
const HOST_PATTERN_OVERRIDE = process.env.SHOWCASE_BACKEND_HOST_PATTERN || "";
|
|
const applyHostPattern = (slug: string, defaultUrl: string): string => {
|
|
if (!HOST_PATTERN_OVERRIDE) return defaultUrl;
|
|
return `https://${HOST_PATTERN_OVERRIDE.replace("{slug}", slug)}`;
|
|
};
|
|
|
|
const rewriteBackendUrl = (slug: string, railwayUrl: string): string => {
|
|
if (USE_LOCAL_PORTS) {
|
|
const port = (localPorts as Record<string, number>)[slug];
|
|
if (!port) throw new Error(`LOCAL_PORTS=1 but no port for slug '${slug}'`);
|
|
return `http://localhost:${port}`;
|
|
}
|
|
return applyHostPattern(slug, railwayUrl);
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Integration registry — derived from showcase/shell/src/data/registry.json
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const INTEGRATIONS = registry.integrations.map((i) => ({
|
|
slug: i.slug,
|
|
name: i.name,
|
|
backendUrl: i.backend_url,
|
|
deployed: i.deployed,
|
|
hasToolRendering: i.features.includes("tool-rendering"),
|
|
demos: i.demos.map((d: { id: string }) => d.id),
|
|
}));
|
|
|
|
// Only test deployed integrations unless SMOKE_ALL=true
|
|
const SMOKE_ALL = process.env.SMOKE_ALL === "true";
|
|
const activeIntegrations = (
|
|
SMOKE_ALL ? INTEGRATIONS : INTEGRATIONS.filter((i) => i.deployed)
|
|
).map((i) => ({ ...i, backendUrl: rewriteBackendUrl(i.slug, i.backendUrl) }));
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Level 1: Health checks (@health) — fast, API-only
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test.describe("Level 1: Backend Health @health", () => {
|
|
for (const integration of activeIntegrations) {
|
|
test(`[L1: health] ${integration.slug} backend is healthy @health`, async ({
|
|
request,
|
|
}) => {
|
|
const result = await checkHealth(request, integration.backendUrl);
|
|
expect(
|
|
result.ok,
|
|
`${integration.slug} health check failed: status=${result.status} path=${result.path} body=${result.body.slice(0, 500)}`,
|
|
).toBe(true);
|
|
});
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Level 2: Agent endpoint reachability (@agent)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test.describe("Level 2: Agent Endpoint @agent", () => {
|
|
for (const integration of activeIntegrations) {
|
|
test(`[L2: agent] ${integration.slug} agent endpoint responds (not 404) @agent`, async ({
|
|
request,
|
|
}) => {
|
|
const result = await checkAgentEndpoint(request, integration.backendUrl);
|
|
expect(
|
|
result.status,
|
|
`${integration.slug} agent endpoint returned 404 — likely using wrong agent type (LangGraphAgent vs HttpAgent). Status=${result.status} body=${result.body.slice(0, 500)}`,
|
|
).not.toBe(404);
|
|
expect(
|
|
result.ok,
|
|
`${integration.slug} agent endpoint is unreachable: status=${result.status} body=${result.body.slice(0, 500)}`,
|
|
).toBe(true);
|
|
});
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Level 3: Round-trip chat (@chat) — browser-based
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test.describe("Level 3: Round-trip Chat @chat", () => {
|
|
for (const integration of activeIntegrations) {
|
|
test(`[L3: chat] ${integration.slug} responds to a message @chat`, async ({
|
|
page,
|
|
}) => {
|
|
test.slow(); // Allow extra time for cold starts
|
|
|
|
const result = await sendChatMessage(
|
|
page,
|
|
integration.backendUrl,
|
|
"Hello, please respond with a brief greeting.",
|
|
"/demos/agentic-chat",
|
|
);
|
|
|
|
expect(
|
|
result.gotResponse,
|
|
`${integration.slug} did not produce an assistant response. The agent may be down, misconfigured, or using the wrong agent type.`,
|
|
).toBe(true);
|
|
expect(
|
|
result.responseText.length,
|
|
`${integration.slug} assistant response was empty`,
|
|
).toBeGreaterThan(0);
|
|
});
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Level 4: Tool rendering (@tools) — browser-based, subset
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test.describe("Level 4: Tool Rendering @tools", () => {
|
|
// Future-proofing: `demos.includes("tool-rendering")` is what actually matters —
|
|
// if a new integration lands without the tool-rendering demo, it's correctly
|
|
// skipped here. The previous `hasToolRendering` boolean was redundant with
|
|
// this check (always `true` alongside the demo entry) and has been removed.
|
|
const toolIntegrations = activeIntegrations.filter((i) =>
|
|
i.demos.includes("tool-rendering"),
|
|
);
|
|
|
|
for (const integration of toolIntegrations) {
|
|
test(`[L4: tools] ${integration.slug} renders tool results @tools`, async ({
|
|
page,
|
|
}) => {
|
|
test.slow();
|
|
|
|
const result = await sendChatMessage(
|
|
page,
|
|
integration.backendUrl,
|
|
"What's the weather in San Francisco?",
|
|
"/demos/tool-rendering",
|
|
);
|
|
|
|
// Tool rendering should produce an assistant response
|
|
expect(
|
|
result.gotResponse,
|
|
`${integration.slug} did not render a tool result`,
|
|
).toBe(true);
|
|
|
|
// The response should contain weather-related content. Avoid a bare
|
|
// `/\d+/` fallback — any digit anywhere makes the assertion vacuous
|
|
// (timestamps, UI chrome, model names all contain digits). Match only
|
|
// on weather-specific vocabulary.
|
|
const responseLC = result.responseText.toLowerCase();
|
|
const hasWeatherContent =
|
|
responseLC.includes("san francisco") ||
|
|
responseLC.includes("weather") ||
|
|
responseLC.includes("temperature") ||
|
|
responseLC.includes("degrees") ||
|
|
responseLC.includes("sunny") ||
|
|
responseLC.includes("cloudy") ||
|
|
responseLC.includes("rain") ||
|
|
responseLC.includes("humidity") ||
|
|
responseLC.includes("wind");
|
|
|
|
expect(
|
|
hasWeatherContent,
|
|
`${integration.slug} response doesn't contain weather info: "${result.responseText}"`,
|
|
).toBe(true);
|
|
|
|
// Check for rendered components (not just text) — catches the query_data
|
|
// loop bug where the agent returns text but never renders a chart/card.
|
|
//
|
|
// isVisible() resolves false for missing/detached elements rather than
|
|
// throwing, so the previous .catch(() => false) was masking nothing on
|
|
// the happy path — but it ALSO masked real errors (e.g. evaluation
|
|
// errors in the locator) as "not visible", which is the concrete bug
|
|
// the original TODO in this file flagged. Drop the swallowing catches
|
|
// and let genuine errors surface as test failures; a missing optional
|
|
// renderer still just yields `false`.
|
|
const responseArea = page
|
|
.locator('[data-testid="copilot-assistant-message"]')
|
|
.last();
|
|
const checks = await Promise.all([
|
|
responseArea.locator(".recharts-wrapper").first().isVisible(),
|
|
responseArea
|
|
.locator("svg circle[stroke-dasharray]")
|
|
.first()
|
|
.isVisible(),
|
|
responseArea
|
|
.locator('[data-testid="weather-card"]')
|
|
.first()
|
|
.isVisible(),
|
|
responseArea.locator("canvas").first().isVisible(),
|
|
]);
|
|
const hasRenderedComponent = checks.some(Boolean);
|
|
|
|
// Warning for now — upgrade to hard failure after data-testid convention
|
|
// is established across all integrations
|
|
if (!hasRenderedComponent) {
|
|
console.warn(
|
|
`\u26a0\ufe0f ${integration.slug}: Tool response contains text but no rendered chart/weather component. This would have missed the query_data loop bug.`,
|
|
);
|
|
}
|
|
});
|
|
}
|
|
});
|