copilotkit--copilotkit/examples/showcases/banking/scripts/memory-drift-smoke.mjs

#!/usr/bin/env node
/**
 * Real-LLM memory drift smoke — NON-GATING, run manually.
 *
 * WHY THIS EXISTS
 * The deterministic CI gate (e2e/memory-learning.spec.ts) serves the agent's
 * LLM from aimock fixtures, so it proves the WIRING (prompt -> tool call -> memory
 * backend -> cross-thread recall -> unlock) but is BLIND to behavioral drift: a
 * fixture replays a fixed decision, so if a prompt edit makes the real model stop
 * calling its memory tools, the aimock test keeps passing. This script closes that
 * gap with a REAL OpenAI call.
 *
 * WHAT IT CHECKS (headless half)
 * It seeds the over-limit procedure as a project/operational memory via REST, then
 * drives a FRESH-THREAD over-limit approval request through the live runtime and
 * asserts the run's event stream contains a `recall_memory` tool call — i.e. the
 * live model still RECALLS-FIRST (the autonomous, load-bearing moment). It also
 * asserts that NO `save_memory` fires on that over-limit request turn (rule 9:
 * GENERAL MEMORY must defer during a procedure). The mid-demonstration turns are
 * HITL/headless-unreachable, so that coverage lives in the aimock e2e + the manual
 * walkthrough.
 *
 * WHAT IT DOES NOT CHECK
 * The SAVE half (the agent emitting `save_memory` after the teach arc) is gated
 * behind the human-in-the-loop teach cards (offerWorkflowRecording ->
 * awaitDashboardDemonstration -> saveLearnedWorkflow) and cannot be driven
 * headlessly. Verify the save path via the manual UI walkthrough (README step 3)
 * and the aimock E2E.
 *
 * REQUIREMENTS
 *   - The memory-enabled stack is up (docker compose; see README) and reachable.
 *   - The demo dev server is running in Intelligence mode (the three INTELLIGENCE_*
 *     env vars set) with a real OPENAI_API_KEY.
 *
 * READINESS GATE
 *   The Intelligence sl-mcp worker can throw an UnhandledPromiseRejection during boot
 *   and briefly drop /mcp connections even after `docker compose up --wait` reports the
 *   container healthy. This smoke first polls `POST /mcp initialize` until it returns 200,
 *   so it only runs once memory is actually serving — a booting/down backend fails fast
 *   with a clear message instead of masquerading as recall drift.
 *
 * USAGE
 *   node scripts/memory-drift-smoke.mjs
 * ENV (optional)
 *   DEMO_URL          default http://localhost:3000
 *   APP_API_URL       default http://localhost:7050
 *   INTELLIGENCE_API_KEY  default cpk_sPRVSEED_seed0privat0longtoken00
 *   CPKI_USER_ID      default jordan-beamson
 *   DRIFT_TXN_ID      default t-3  (an over-limit pending seed txn)
 *
 * NOTE: the run is posted to the AG-UI run endpoint
 * `${DEMO_URL}/api/copilotkit/agent/default/run` with a minimal RunAgentInput. If a
 * future runtime version changes that path or body shape, that POST is the one spot
 * to adjust — the seed/recall REST calls and the stream scan are stable.
 */

const DEMO_URL = process.env.DEMO_URL ?? "http://localhost:3000";
const APP_API_URL = process.env.APP_API_URL ?? "http://localhost:7050";
const KEY =
  process.env.INTELLIGENCE_API_KEY ?? "cpk_sPRVSEED_seed0privat0longtoken00";
const USER_ID = process.env.CPKI_USER_ID ?? "jordan-beamson";
const TXN_ID = process.env.DRIFT_TXN_ID ?? "t-3";

const PROCEDURE = (code) =>
  `To approve an over-limit charge, open a policy exception with code ${code} ` +
  `against the charge and finalize it, then approve the transaction.`;
const SEED_CODE = "EXC-BOARD-APPROVED";

function log(ok, msg) {
  console.log(`${ok ? "✓" : "✗"} ${msg}`);
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// Poll POST /mcp `initialize` until the sl-mcp worker answers 200 and the SSE body
// completes without a reset. Guards against the Intelligence backend's boot window,
// where the worker throws an UnhandledPromiseRejection and drops /mcp connections even
// though the container reports healthy — running against that window makes the agent
// lose recall_memory mid-run and looks like drift. Fails fast so a booting backend
// never masquerades as a prompt regression.
async function waitForMcpReady({ retries = 30, delayMs = 1000 } = {}) {
  const body = JSON.stringify({
    jsonrpc: "2.0",
    id: 1,
    method: "initialize",
    params: {
      protocolVersion: "2025-11-25",
      capabilities: {},
      clientInfo: { name: "smoke-preflight", version: "1" },
    },
  });
  for (let i = 0; i < retries; i++) {
    try {
      const res = await fetch(`${APP_API_URL}/mcp`, {
        method: "POST",
        headers: {
          Authorization: `Bearer ${KEY}`,
          "X-Cpki-User-Id": USER_ID,
          "Content-Type": "application/json",
          Accept: "application/json, text/event-stream",
          "mcp-protocol-version": "2025-11-25",
        },
        body,
      });
      if (res.ok) {
        await res.text(); // reading the body catches a mid-stream reset
        return;
      }
    } catch {
      // connection refused / reset during boot — keep polling
    }
    await sleep(delayMs);
  }
  throw new Error(
    `MCP not ready after ${retries * delayMs}ms — the Intelligence sl-mcp worker never ` +
      `stabilized at POST ${APP_API_URL}/mcp (initialize). Bring up / restart the stack ` +
      `(docker compose up -d --wait) and confirm 'docker logs' shows no boot-time ` +
      `UnhandledPromiseRejection, then retry.`,
  );
}

// Confirm the demo dev server (pnpm dev) is up at DEMO_URL. The /mcp gate only covers
// the backend (:7050); the over-limit run below hits the app (:3000). Any HTTP response
// means it is serving; only a connection error means it is down.
async function waitForDemoServer({ retries = 20, delayMs = 1000 } = {}) {
  for (let i = 0; i < retries; i++) {
    try {
      await fetch(DEMO_URL, { method: "GET" });
      return;
    } catch {
      // connection refused — dev server not up yet
    }
    await sleep(delayMs);
  }
  throw new Error(
    `Demo dev server not reachable at ${DEMO_URL} — start it with 'pnpm dev' ` +
      `(Intelligence mode: the three INTELLIGENCE_* env vars + a real OPENAI_API_KEY), then retry.`,
  );
}

async function seedProcedureMemory() {
  const res = await fetch(`${APP_API_URL}/api/memories`, {
    method: "POST",
    headers: {
      Authorization: `Bearer ${KEY}`,
      "X-Cpki-User-Id": USER_ID,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({
      content: PROCEDURE(SEED_CODE),
      scope: "project",
      kind: "operational",
    }),
  });
  if (!res.ok)
    throw new Error(
      `seed memory failed: HTTP ${res.status} ${await res.text()}`,
    );
  const body = await res.json().catch(() => ({}));
  return body;
}

async function runOverLimitTurn() {
  // Minimal AG-UI RunAgentInput. A fresh UUID threadId guarantees no in-thread context
  // (the only way the agent can know the procedure is by calling recall_memory) and
  // satisfies the Intelligence backend's UUID validation (a custom "drift-..." id 400s).
  const threadId = crypto.randomUUID();
  const body = {
    threadId,
    runId: crypto.randomUUID(),
    state: {},
    // Alex -> jordan-beamson (the id we seed the procedure under). Makes the
    // smoke identity-self-sufficient so it passes against the unpinned live demo.
    properties: { userId: "9g5h2j1k4l", userRole: "Admin" },
    messages: [
      {
        id: "m1",
        role: "user",
        content: `Please approve the over-limit charge ${TXN_ID}.`,
      },
    ],
    tools: [],
    context: [],
    forwardedProps: {},
  };
  const res = await fetch(`${DEMO_URL}/api/copilotkit/agent/default/run`, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Accept: "text/event-stream",
    },
    body: JSON.stringify(body),
  });
  if (!res.ok) {
    throw new Error(
      `run POST failed: HTTP ${res.status} ${await res.text().catch(() => "")}\n` +
        "hint: confirm the demo is running in Intelligence mode and the run endpoint " +
        "path/body shape matches this runtime version (see header NOTE).",
    );
  }
  // Drain the FULL turn's SSE (until the run ends or the deadline) before scanning.
  // We must not early-return on the first recall_memory frame: because RECALL FIRST
  // makes recall stream before anything else, a spurious general save_memory (rule 9
  // leak) streams AFTER it — cancelling the reader on recall would cut that frame off
  // and make the negative-save assertion a false negative. The turn ends after the
  // agent emits its tool calls (the HITL cards are answered on the NEXT turn, which
  // we never send), so draining terminates naturally; the deadline is a backstop.
  const reader = res.body.getReader();
  const decoder = new TextDecoder();
  let buf = "";
  const deadline = Date.now() + 60_000;
  while (Date.now() < deadline) {
    const { value, done } = await reader.read();
    if (done) break;
    buf += decoder.decode(value, { stream: true });
  }
  reader.cancel().catch(() => {});
  return {
    recalled: /recall_memory/.test(buf),
    sawSave: /save_memory/.test(buf),
  };
}

console.log(
  `memory drift smoke (REAL LLM) — demo ${DEMO_URL}, app-api ${APP_API_URL}, txn ${TXN_ID}\n`,
);

try {
  await waitForMcpReady();
  log(true, "preflight: /mcp initialize is serving (memory tools ready)");
  await waitForDemoServer();
  log(true, `preflight: demo dev server reachable at ${DEMO_URL}`);

  const seeded = await seedProcedureMemory();
  log(
    true,
    `seeded project/operational procedure memory (${seeded.absorbed ? "absorbed" : "created"})`,
  );

  const { recalled, sawSave } = await runOverLimitTurn();
  log(
    recalled,
    recalled
      ? "PASS: live model emitted recall_memory on a fresh-thread over-limit request"
      : "DRIFT: live model did NOT emit recall_memory — the recall-first prompt may have regressed",
  );
  // Rule 9 (DEFER DURING PROCEDURES): the over-limit request turn must NOT emit a
  // general save. A spurious save_memory here means the GENERAL MEMORY block is
  // firing inside the teach flow.
  log(
    !sawSave,
    sawSave
      ? "DRIFT: a save_memory fired on the over-limit request turn — GENERAL MEMORY leaked into the procedure (rule 9)"
      : "PASS: no spurious save_memory on the over-limit request turn",
  );
  process.exit(recalled && !sawSave ? 0 : 1);
} catch (err) {
  log(false, `error: ${String(err).slice(0, 400)}`);
  process.exit(2);
}