wehub-resource-sync
116 lines
4.9 KiB
TypeScript
116 lines
4.9 KiB
TypeScript
import { test, expect } from "@playwright/test";
|
|
import type { Page } from "@playwright/test";
|
|
|
|
// QA reference: qa/a2ui-recovery.md
|
|
// Demo source: src/app/demos/a2ui-recovery/{page.tsx, chat.tsx, suggestions.ts}
|
|
//
|
|
// A2UI error recovery (OSS-158 / OSS-375). Assert the STABLE end-states — the
|
|
// recovered surface paints (heal) and the hard-failure UI shows (exhaust) — and
|
|
// deliberately do NOT assert the transient "Retrying generation… (N/M)" label,
|
|
// which is threshold-gated + timing dependent (see @copilotkit/react-core/v2
|
|
// A2UIRecoveryStates) and would be flaky.
|
|
//
|
|
// The aimock fixtures (showcase/aimock/d6/langgraph-python/a2ui-recovery.json)
|
|
// drive the inner render_a2ui sub-agent two ways: HEAL emits free-form/sloppy
|
|
// args (components/data as JSON strings) the middleware heals via parse_and_fix;
|
|
// EXHAUST is structurally invalid (unresolved child) on every attempt, so the
|
|
// validate->retry loop hits the cap and returns the a2ui_recovery_exhausted
|
|
// envelope. Healing + the loop run live in the backend ag_ui_langgraph
|
|
// get_a2ui_tools factory (injectA2UITool=false); the failure UI text ("Couldn't
|
|
// generate the UI") comes from @copilotkit/react-core/v2. The recovered surface
|
|
// reuses the declarative-gen-ui catalog, so it carries the `declarative-metric`
|
|
// testid.
|
|
//
|
|
// Requires the stack running with aimock so the malformed renders fire
|
|
// deterministically; against a real LLM the demo would not reliably produce the
|
|
// invalid attempts.
|
|
|
|
/** Click a suggestion pill and confirm the message dispatched (the user
|
|
* bubble with the pill's full message text appears). The user bubble is
|
|
* matched by `data-testid="copilot-user-message"`. Slow hydration can swallow
|
|
* the first click, so we retry; never re-click once dispatched. */
|
|
async function clickPill(page: Page, title: string, message: string) {
|
|
const pill = page
|
|
.locator('[data-testid="copilot-suggestion"]')
|
|
.filter({ hasText: title })
|
|
.first();
|
|
await expect(pill).toBeVisible({ timeout: 15_000 });
|
|
const userBubble = page
|
|
.locator('[data-testid="copilot-user-message"]')
|
|
.filter({ hasText: message })
|
|
.first();
|
|
await expect(async () => {
|
|
if ((await userBubble.count()) === 0) {
|
|
await pill.click();
|
|
}
|
|
await expect(userBubble).toBeVisible({ timeout: 3_000 });
|
|
}).toPass({ timeout: 30_000 });
|
|
}
|
|
|
|
const HEAL_PILL = "Recover a bad render";
|
|
const HEAL_MSG =
|
|
"Build my Q2 revenue summary and self-correct a malformed first attempt.";
|
|
const EXHAUST_PILL = "Show an unrecoverable failure";
|
|
const EXHAUST_MSG =
|
|
"Build a report that fails every validation pass so I can preview the fallback.";
|
|
|
|
test.describe("A2UI Error Recovery", () => {
|
|
test.setTimeout(120_000);
|
|
|
|
test.beforeEach(async ({ page }) => {
|
|
await page.goto("/demos/a2ui-recovery");
|
|
});
|
|
|
|
test("page loads with both recovery pills", async ({ page }) => {
|
|
await expect(page.getByPlaceholder("Type a message")).toBeVisible();
|
|
const suggestions = page.locator('[data-testid="copilot-suggestion"]');
|
|
for (const title of [HEAL_PILL, EXHAUST_PILL]) {
|
|
await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({
|
|
timeout: 15_000,
|
|
});
|
|
}
|
|
// No surface on first paint.
|
|
await expect(page.getByTestId("declarative-metric")).toHaveCount(0);
|
|
});
|
|
|
|
test("heal: free-form/sloppy render is healed into a valid surface", async ({
|
|
page,
|
|
}) => {
|
|
await clickPill(page, HEAL_PILL, HEAL_MSG);
|
|
|
|
// The model emits free-form (stringified) A2UI args; the middleware heals
|
|
// them via parse_and_fix into a valid surface that paints. Allow 90s for
|
|
// the sub-agent round-trip.
|
|
const metrics = page.locator('[data-testid="declarative-metric"]');
|
|
await expect
|
|
.poll(async () => await metrics.count(), { timeout: 90_000 })
|
|
.toBeGreaterThanOrEqual(2);
|
|
|
|
// The healed surface, NOT the hard-failure UI, and no render-error banners.
|
|
await expect(page.getByText("Couldn't generate the UI")).toHaveCount(0);
|
|
await expect(page.getByText(/Catalog not found/i)).toHaveCount(0);
|
|
await expect(
|
|
page.getByText(/Cannot create component .* without a type/i),
|
|
).toHaveCount(0);
|
|
});
|
|
|
|
test("exhaust: always-invalid render shows the hard-failure UI, no faulty surface", async ({
|
|
page,
|
|
}) => {
|
|
await clickPill(page, EXHAUST_PILL, EXHAUST_MSG);
|
|
|
|
// Hard-failure UI appears once the attempt cap is hit (A2UIRecoveryStates
|
|
// renders this on status: "failed" = the a2ui_recovery_exhausted envelope).
|
|
await expect(
|
|
page.getByText("Couldn't generate the UI").first(),
|
|
).toBeVisible({ timeout: 90_000 });
|
|
|
|
// No faulty surface ever paints (server-side no-wipe guarantee: middleware
|
|
// gate + adapter recovery loop).
|
|
await expect(page.getByTestId("declarative-metric")).toHaveCount(0);
|
|
|
|
// Conversation remains usable after the hard failure.
|
|
await expect(page.getByPlaceholder("Type a message")).toBeEnabled();
|
|
});
|
|
});
|