chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Real-LLM memory drift smoke — NON-GATING, run manually.
|
||||
*
|
||||
* WHY THIS EXISTS
|
||||
* The deterministic CI gate (e2e/memory-learning.spec.ts) serves the agent's
|
||||
* LLM from aimock fixtures, so it proves the WIRING (prompt -> tool call -> memory
|
||||
* backend -> cross-thread recall -> unlock) but is BLIND to behavioral drift: a
|
||||
* fixture replays a fixed decision, so if a prompt edit makes the real model stop
|
||||
* calling its memory tools, the aimock test keeps passing. This script closes that
|
||||
* gap with a REAL OpenAI call.
|
||||
*
|
||||
* WHAT IT CHECKS (headless half)
|
||||
* It seeds the over-limit procedure as a project/operational memory via REST, then
|
||||
* drives a FRESH-THREAD over-limit approval request through the live runtime and
|
||||
* asserts the run's event stream contains a `recall_memory` tool call — i.e. the
|
||||
* live model still RECALLS-FIRST (the autonomous, load-bearing moment). It also
|
||||
* asserts that NO `save_memory` fires on that over-limit request turn (rule 9:
|
||||
* GENERAL MEMORY must defer during a procedure). The mid-demonstration turns are
|
||||
* HITL/headless-unreachable, so that coverage lives in the aimock e2e + the manual
|
||||
* walkthrough.
|
||||
*
|
||||
* WHAT IT DOES NOT CHECK
|
||||
* The SAVE half (the agent emitting `save_memory` after the teach arc) is gated
|
||||
* behind the human-in-the-loop teach cards (offerWorkflowRecording ->
|
||||
* awaitDashboardDemonstration -> saveLearnedWorkflow) and cannot be driven
|
||||
* headlessly. Verify the save path via the manual UI walkthrough (README step 3)
|
||||
* and the aimock E2E.
|
||||
*
|
||||
* REQUIREMENTS
|
||||
* - The memory-enabled stack is up (docker compose; see README) and reachable.
|
||||
* - The demo dev server is running in Intelligence mode (the three INTELLIGENCE_*
|
||||
* env vars set) with a real OPENAI_API_KEY.
|
||||
*
|
||||
* READINESS GATE
|
||||
* The Intelligence sl-mcp worker can throw an UnhandledPromiseRejection during boot
|
||||
* and briefly drop /mcp connections even after `docker compose up --wait` reports the
|
||||
* container healthy. This smoke first polls `POST /mcp initialize` until it returns 200,
|
||||
* so it only runs once memory is actually serving — a booting/down backend fails fast
|
||||
* with a clear message instead of masquerading as recall drift.
|
||||
*
|
||||
* USAGE
|
||||
* node scripts/memory-drift-smoke.mjs
|
||||
* ENV (optional)
|
||||
* DEMO_URL default http://localhost:3000
|
||||
* APP_API_URL default http://localhost:7050
|
||||
* INTELLIGENCE_API_KEY default cpk_sPRVSEED_seed0privat0longtoken00
|
||||
* CPKI_USER_ID default jordan-beamson
|
||||
* DRIFT_TXN_ID default t-3 (an over-limit pending seed txn)
|
||||
*
|
||||
* NOTE: the run is posted to the AG-UI run endpoint
|
||||
* `${DEMO_URL}/api/copilotkit/agent/default/run` with a minimal RunAgentInput. If a
|
||||
* future runtime version changes that path or body shape, that POST is the one spot
|
||||
* to adjust — the seed/recall REST calls and the stream scan are stable.
|
||||
*/
|
||||
|
||||
const DEMO_URL = process.env.DEMO_URL ?? "http://localhost:3000";
|
||||
const APP_API_URL = process.env.APP_API_URL ?? "http://localhost:7050";
|
||||
const KEY =
|
||||
process.env.INTELLIGENCE_API_KEY ?? "cpk_sPRVSEED_seed0privat0longtoken00";
|
||||
const USER_ID = process.env.CPKI_USER_ID ?? "jordan-beamson";
|
||||
const TXN_ID = process.env.DRIFT_TXN_ID ?? "t-3";
|
||||
|
||||
const PROCEDURE = (code) =>
|
||||
`To approve an over-limit charge, open a policy exception with code ${code} ` +
|
||||
`against the charge and finalize it, then approve the transaction.`;
|
||||
const SEED_CODE = "EXC-BOARD-APPROVED";
|
||||
|
||||
function log(ok, msg) {
|
||||
console.log(`${ok ? "✓" : "✗"} ${msg}`);
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// Poll POST /mcp `initialize` until the sl-mcp worker answers 200 and the SSE body
|
||||
// completes without a reset. Guards against the Intelligence backend's boot window,
|
||||
// where the worker throws an UnhandledPromiseRejection and drops /mcp connections even
|
||||
// though the container reports healthy — running against that window makes the agent
|
||||
// lose recall_memory mid-run and looks like drift. Fails fast so a booting backend
|
||||
// never masquerades as a prompt regression.
|
||||
async function waitForMcpReady({ retries = 30, delayMs = 1000 } = {}) {
|
||||
const body = JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: 1,
|
||||
method: "initialize",
|
||||
params: {
|
||||
protocolVersion: "2025-11-25",
|
||||
capabilities: {},
|
||||
clientInfo: { name: "smoke-preflight", version: "1" },
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
const res = await fetch(`${APP_API_URL}/mcp`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${KEY}`,
|
||||
"X-Cpki-User-Id": USER_ID,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/event-stream",
|
||||
"mcp-protocol-version": "2025-11-25",
|
||||
},
|
||||
body,
|
||||
});
|
||||
if (res.ok) {
|
||||
await res.text(); // reading the body catches a mid-stream reset
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// connection refused / reset during boot — keep polling
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
throw new Error(
|
||||
`MCP not ready after ${retries * delayMs}ms — the Intelligence sl-mcp worker never ` +
|
||||
`stabilized at POST ${APP_API_URL}/mcp (initialize). Bring up / restart the stack ` +
|
||||
`(docker compose up -d --wait) and confirm 'docker logs' shows no boot-time ` +
|
||||
`UnhandledPromiseRejection, then retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Confirm the demo dev server (pnpm dev) is up at DEMO_URL. The /mcp gate only covers
|
||||
// the backend (:7050); the over-limit run below hits the app (:3000). Any HTTP response
|
||||
// means it is serving; only a connection error means it is down.
|
||||
async function waitForDemoServer({ retries = 20, delayMs = 1000 } = {}) {
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
await fetch(DEMO_URL, { method: "GET" });
|
||||
return;
|
||||
} catch {
|
||||
// connection refused — dev server not up yet
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
throw new Error(
|
||||
`Demo dev server not reachable at ${DEMO_URL} — start it with 'pnpm dev' ` +
|
||||
`(Intelligence mode: the three INTELLIGENCE_* env vars + a real OPENAI_API_KEY), then retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
async function seedProcedureMemory() {
|
||||
const res = await fetch(`${APP_API_URL}/api/memories`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${KEY}`,
|
||||
"X-Cpki-User-Id": USER_ID,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
content: PROCEDURE(SEED_CODE),
|
||||
scope: "project",
|
||||
kind: "operational",
|
||||
}),
|
||||
});
|
||||
if (!res.ok)
|
||||
throw new Error(
|
||||
`seed memory failed: HTTP ${res.status} ${await res.text()}`,
|
||||
);
|
||||
const body = await res.json().catch(() => ({}));
|
||||
return body;
|
||||
}
|
||||
|
||||
async function runOverLimitTurn() {
|
||||
// Minimal AG-UI RunAgentInput. A fresh UUID threadId guarantees no in-thread context
|
||||
// (the only way the agent can know the procedure is by calling recall_memory) and
|
||||
// satisfies the Intelligence backend's UUID validation (a custom "drift-..." id 400s).
|
||||
const threadId = crypto.randomUUID();
|
||||
const body = {
|
||||
threadId,
|
||||
runId: crypto.randomUUID(),
|
||||
state: {},
|
||||
// Alex -> jordan-beamson (the id we seed the procedure under). Makes the
|
||||
// smoke identity-self-sufficient so it passes against the unpinned live demo.
|
||||
properties: { userId: "9g5h2j1k4l", userRole: "Admin" },
|
||||
messages: [
|
||||
{
|
||||
id: "m1",
|
||||
role: "user",
|
||||
content: `Please approve the over-limit charge ${TXN_ID}.`,
|
||||
},
|
||||
],
|
||||
tools: [],
|
||||
context: [],
|
||||
forwardedProps: {},
|
||||
};
|
||||
const res = await fetch(`${DEMO_URL}/api/copilotkit/agent/default/run`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`run POST failed: HTTP ${res.status} ${await res.text().catch(() => "")}\n` +
|
||||
"hint: confirm the demo is running in Intelligence mode and the run endpoint " +
|
||||
"path/body shape matches this runtime version (see header NOTE).",
|
||||
);
|
||||
}
|
||||
// Drain the FULL turn's SSE (until the run ends or the deadline) before scanning.
|
||||
// We must not early-return on the first recall_memory frame: because RECALL FIRST
|
||||
// makes recall stream before anything else, a spurious general save_memory (rule 9
|
||||
// leak) streams AFTER it — cancelling the reader on recall would cut that frame off
|
||||
// and make the negative-save assertion a false negative. The turn ends after the
|
||||
// agent emits its tool calls (the HITL cards are answered on the NEXT turn, which
|
||||
// we never send), so draining terminates naturally; the deadline is a backstop.
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
const deadline = Date.now() + 60_000;
|
||||
while (Date.now() < deadline) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
}
|
||||
reader.cancel().catch(() => {});
|
||||
return {
|
||||
recalled: /recall_memory/.test(buf),
|
||||
sawSave: /save_memory/.test(buf),
|
||||
};
|
||||
}
|
||||
|
||||
console.log(
|
||||
`memory drift smoke (REAL LLM) — demo ${DEMO_URL}, app-api ${APP_API_URL}, txn ${TXN_ID}\n`,
|
||||
);
|
||||
|
||||
try {
|
||||
await waitForMcpReady();
|
||||
log(true, "preflight: /mcp initialize is serving (memory tools ready)");
|
||||
await waitForDemoServer();
|
||||
log(true, `preflight: demo dev server reachable at ${DEMO_URL}`);
|
||||
|
||||
const seeded = await seedProcedureMemory();
|
||||
log(
|
||||
true,
|
||||
`seeded project/operational procedure memory (${seeded.absorbed ? "absorbed" : "created"})`,
|
||||
);
|
||||
|
||||
const { recalled, sawSave } = await runOverLimitTurn();
|
||||
log(
|
||||
recalled,
|
||||
recalled
|
||||
? "PASS: live model emitted recall_memory on a fresh-thread over-limit request"
|
||||
: "DRIFT: live model did NOT emit recall_memory — the recall-first prompt may have regressed",
|
||||
);
|
||||
// Rule 9 (DEFER DURING PROCEDURES): the over-limit request turn must NOT emit a
|
||||
// general save. A spurious save_memory here means the GENERAL MEMORY block is
|
||||
// firing inside the teach flow.
|
||||
log(
|
||||
!sawSave,
|
||||
sawSave
|
||||
? "DRIFT: a save_memory fired on the over-limit request turn — GENERAL MEMORY leaked into the procedure (rule 9)"
|
||||
: "PASS: no spurious save_memory on the over-limit request turn",
|
||||
);
|
||||
process.exit(recalled && !sawSave ? 0 : 1);
|
||||
} catch (err) {
|
||||
log(false, `error: ${String(err).slice(0, 400)}`);
|
||||
process.exit(2);
|
||||
}
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Real-LLM general-memory smoke — NON-GATING, run manually.
|
||||
* Mirrors memory-drift-smoke.mjs. Asserts the GENERAL MEMORY prompt behavior:
|
||||
* SAVE — a personal fact triggers save_memory(kind:"topical", scope:"user")
|
||||
* NO-SAVE — a secret is NOT saved
|
||||
* RECALL — a seeded user-scoped fact is recalled on a fresh thread
|
||||
* ISOLATION — a user-scoped fact does NOT cross to a different seeded user;
|
||||
* a project-scoped fact DOES (REST-level, no LLM ranking dependency)
|
||||
*
|
||||
* REQUIREMENTS: the memory stack is up and the demo runs in Intelligence mode
|
||||
* with a real OPENAI_API_KEY (see README). Uses two seeded users:
|
||||
* ALEX_ID -> jordan-beamson, MAYA_ID -> morgan-fluxx.
|
||||
*
|
||||
* READINESS GATE: the Intelligence sl-mcp worker can throw an UnhandledPromiseRejection
|
||||
* during boot and briefly drop /mcp connections even after `docker compose up --wait`
|
||||
* reports the container healthy. Running against that window makes the agent lose its
|
||||
* memory tools mid-run and surfaces as unrelated failures. This smoke first polls
|
||||
* `POST /mcp initialize` until it returns 200, so it only runs once memory is actually
|
||||
* serving. If you see connection resets or "no fixture matched" style flakes, it is the
|
||||
* backend startup window, not this script.
|
||||
*/
|
||||
const DEMO_URL = process.env.DEMO_URL ?? "http://localhost:3000";
|
||||
const APP_API_URL = process.env.APP_API_URL ?? "http://localhost:7050";
|
||||
const KEY =
|
||||
process.env.INTELLIGENCE_API_KEY ?? "cpk_sPRVSEED_seed0privat0longtoken00";
|
||||
const ALEX = {
|
||||
memberId: "9g5h2j1k4l",
|
||||
role: "Admin",
|
||||
userId: "jordan-beamson",
|
||||
};
|
||||
const MAYA = { userId: "morgan-fluxx" };
|
||||
|
||||
function log(ok, msg) {
|
||||
console.log(`${ok ? "✓" : "✗"} ${msg}`);
|
||||
}
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// Poll POST /mcp `initialize` until the sl-mcp worker answers 200 and the SSE body
|
||||
// completes without a reset. Guards against the Intelligence backend's boot window,
|
||||
// where the worker throws an UnhandledPromiseRejection and drops /mcp connections
|
||||
// even though the container reports healthy. Fails fast with a clear message so a
|
||||
// down/booting backend never masquerades as a feature regression.
|
||||
async function waitForMcpReady({ retries = 30, delayMs = 1000 } = {}) {
|
||||
const body = JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: 1,
|
||||
method: "initialize",
|
||||
params: {
|
||||
protocolVersion: "2025-11-25",
|
||||
capabilities: {},
|
||||
clientInfo: { name: "smoke-preflight", version: "1" },
|
||||
},
|
||||
});
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
const res = await fetch(`${APP_API_URL}/mcp`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${KEY}`,
|
||||
"X-Cpki-User-Id": ALEX.userId,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/event-stream",
|
||||
"mcp-protocol-version": "2025-11-25",
|
||||
},
|
||||
body,
|
||||
});
|
||||
if (res.ok) {
|
||||
await res.text();
|
||||
return;
|
||||
} // reading the body catches a mid-stream reset
|
||||
} catch {
|
||||
// connection refused / reset during boot — keep polling
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
throw new Error(
|
||||
`MCP not ready after ${retries * delayMs}ms — the Intelligence sl-mcp worker never ` +
|
||||
`stabilized at POST ${APP_API_URL}/mcp (initialize). Bring up / restart the stack ` +
|
||||
`(docker compose up -d --wait) and confirm 'docker logs' shows no boot-time ` +
|
||||
`UnhandledPromiseRejection, then retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
async function restSave(userId, content, kind, scope) {
|
||||
const res = await fetch(`${APP_API_URL}/api/memories`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${KEY}`,
|
||||
"X-Cpki-User-Id": userId,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ content, kind, scope }),
|
||||
});
|
||||
if (!res.ok)
|
||||
throw new Error(`save failed HTTP ${res.status} ${await res.text()}`);
|
||||
return res.json().catch(() => ({}));
|
||||
}
|
||||
|
||||
async function restRecall(userId, query, scope) {
|
||||
const res = await fetch(`${APP_API_URL}/api/memories/recall`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${KEY}`,
|
||||
"X-Cpki-User-Id": userId,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(scope ? { query, scope } : { query }),
|
||||
});
|
||||
if (!res.ok)
|
||||
throw new Error(`recall failed HTTP ${res.status} ${await res.text()}`);
|
||||
const body = await res.json().catch(() => ({ memories: [] }));
|
||||
return body.memories ?? [];
|
||||
}
|
||||
|
||||
// Confirm the demo dev server (pnpm dev) is actually up at DEMO_URL. The /mcp gate
|
||||
// only covers the backend (:7050); the chat turns below hit the app (:3000). Any HTTP
|
||||
// response (even 404) means it is serving; only a connection error means it is down.
|
||||
async function waitForDemoServer({ retries = 20, delayMs = 1000 } = {}) {
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
await fetch(DEMO_URL, { method: "GET" });
|
||||
return;
|
||||
} catch {
|
||||
// connection refused — dev server not up yet
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
throw new Error(
|
||||
`Demo dev server not reachable at ${DEMO_URL} — start it with 'pnpm dev' ` +
|
||||
`(Intelligence mode: the three INTELLIGENCE_* env vars + a real OPENAI_API_KEY), then retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Drive one chat turn as Alex; return the concatenated SSE buffer so callers can
|
||||
// scan for tool-call names and, for a personal fact, the kind/scope args.
|
||||
async function turn(content) {
|
||||
// A fresh UUID thread per turn: the Intelligence backend validates thread ids as
|
||||
// UUIDs (a custom "facts-..." id 400s), and a new thread guarantees no in-thread
|
||||
// context so cross-thread recall is the only way the agent can know a prior fact.
|
||||
const threadId = crypto.randomUUID();
|
||||
const res = await fetch(`${DEMO_URL}/api/copilotkit/agent/default/run`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
threadId,
|
||||
runId: crypto.randomUUID(),
|
||||
state: {},
|
||||
properties: { userId: ALEX.memberId, userRole: ALEX.role },
|
||||
messages: [{ id: "m1", role: "user", content }],
|
||||
tools: [],
|
||||
context: [],
|
||||
forwardedProps: {},
|
||||
}),
|
||||
});
|
||||
if (!res.ok)
|
||||
throw new Error(
|
||||
`run failed HTTP ${res.status} ${await res.text().catch(() => "")}`,
|
||||
);
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
const deadline = Date.now() + 60_000;
|
||||
while (Date.now() < deadline) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`memory facts smoke (REAL LLM) — demo ${DEMO_URL}, app-api ${APP_API_URL}\n`,
|
||||
);
|
||||
let failures = 0;
|
||||
const check = (ok, msg) => {
|
||||
log(ok, msg);
|
||||
if (!ok) failures++;
|
||||
};
|
||||
|
||||
try {
|
||||
await waitForMcpReady();
|
||||
log(true, "preflight: /mcp initialize is serving (memory tools ready)");
|
||||
await waitForDemoServer();
|
||||
log(true, `preflight: demo dev server reachable at ${DEMO_URL}`);
|
||||
|
||||
// SAVE: a personal fact must trigger save_memory with kind:"topical", scope:"user".
|
||||
const saveBuf = await turn("remember my favorite food is sushi");
|
||||
const saved = /save_memory/.test(saveBuf);
|
||||
const topicalUser =
|
||||
/"kind"\s*:\s*"topical"/.test(saveBuf) &&
|
||||
/"scope"\s*:\s*"user"/.test(saveBuf);
|
||||
check(saved, "SAVE: save_memory fired for a personal fact");
|
||||
check(
|
||||
topicalUser,
|
||||
"SAVE: save carried kind:topical scope:user (cross-thread recallable)",
|
||||
);
|
||||
|
||||
// NO-SAVE: a secret must NOT be saved.
|
||||
const secretBuf = await turn("remember my API key is sk-abc123");
|
||||
check(!/save_memory/.test(secretBuf), "NO-SAVE: no save_memory for a secret");
|
||||
|
||||
// RECALL cross-thread: seed a user fact via REST, then ask on a fresh thread.
|
||||
await restSave(
|
||||
ALEX.userId,
|
||||
"office is in the Berlin branch",
|
||||
"topical",
|
||||
"user",
|
||||
);
|
||||
const recallBuf = await turn("where is my office?");
|
||||
check(
|
||||
/recall_memory/.test(recallBuf),
|
||||
"RECALL: recall_memory fired on a fresh thread",
|
||||
);
|
||||
|
||||
// ISOLATION (REST-level, deterministic): seed user + project facts under Alex,
|
||||
// recall as Maya. Project crosses; user does not.
|
||||
await restSave(ALEX.userId, "favorite food is sushi", "topical", "user");
|
||||
await restSave(
|
||||
ALEX.userId,
|
||||
"our fiscal year ends in March",
|
||||
"topical",
|
||||
"project",
|
||||
);
|
||||
const mayaProject = await restRecall(
|
||||
MAYA.userId,
|
||||
"fiscal year end",
|
||||
"project",
|
||||
);
|
||||
const mayaUser = await restRecall(MAYA.userId, "favorite food", "user");
|
||||
check(
|
||||
mayaProject.some((m) => /march/i.test(m.content)),
|
||||
"ISOLATION: project fact crosses to the other user",
|
||||
);
|
||||
check(
|
||||
!mayaUser.some((m) => /sushi/i.test(m.content)),
|
||||
"ISOLATION: user fact does NOT cross to the other user",
|
||||
);
|
||||
|
||||
process.exit(failures === 0 ? 0 : 1);
|
||||
} catch (err) {
|
||||
log(false, `error: ${String(err).slice(0, 400)}`);
|
||||
process.exit(2);
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* mint-dev-license — DEV-ONLY helper for the SELF-HOSTED Intelligence stack.
|
||||
*
|
||||
* Prints (or writes to .env) the two env values that unlock the paid `memory`
|
||||
* feature on a *locally-built* self-hosted Intelligence stack:
|
||||
*
|
||||
* COPILOTKIT_LICENSE_TOKEN=<signed enterprise dev license, features.memory=true>
|
||||
* BAKED_LICENSE_KEYS_JSON={"<keyId>":"<publicKey>"}
|
||||
*
|
||||
* WHY THIS EXISTS (and when you do NOT need it)
|
||||
* --------------------------------------------
|
||||
* Self-hosted Intelligence gates memory behind a signed offline license. A
|
||||
* *locally-built* (unbaked) app-api reads BAKED_LICENSE_KEYS_JSON live at
|
||||
* runtime, so a throwaway keypair whose public half is baked in can sign a
|
||||
* license the verifier trusts — no master-key attestation needed. See
|
||||
* packages/license-verifier/src/keystore.ts in the Intelligence repo.
|
||||
*
|
||||
* This does NOT work against — and is NOT needed for — MANAGED Intelligence or
|
||||
* the official public GHCR images: those bake CopilotKit's master public key as
|
||||
* the immutable root of trust and ignore a runtime BAKED_LICENSE_KEYS_JSON. For
|
||||
* the managed path you set a CopilotKit-ISSUED COPILOTKIT_LICENSE_TOKEN and OMIT
|
||||
* BAKED_LICENSE_KEYS_JSON entirely (see .env.example). This script is purely a
|
||||
* local-dev convenience and is never imported by the app runtime, so it does not
|
||||
* couple the demo to the self-hosted stack.
|
||||
*
|
||||
* The signer lives in the private Intelligence source (it depends on the
|
||||
* @cpki/license-catalog workspace package), so this wrapper drives that repo's
|
||||
* own toolchain rather than vendoring any signing code into this public repo.
|
||||
* Point INTELLIGENCE_REPO at your Intelligence checkout (defaults to the sibling
|
||||
* path the docker-compose image build also uses).
|
||||
*
|
||||
* USAGE
|
||||
* node scripts/mint-dev-license.mjs # print the two env lines
|
||||
* node scripts/mint-dev-license.mjs --write # upsert them into ./.env
|
||||
* INTELLIGENCE_REPO=/path/to/Intelligence node scripts/mint-dev-license.mjs
|
||||
* node scripts/mint-dev-license.mjs --org my-org # override the license org
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
|
||||
const DEMO_ROOT = resolve(SCRIPT_DIR, "..");
|
||||
|
||||
// Match the docker-compose image build default: ${INTELLIGENCE_REPO:-../../../../Intelligence}
|
||||
const DEFAULT_INTELLIGENCE_REPO = resolve(
|
||||
DEMO_ROOT,
|
||||
"../../../../Intelligence",
|
||||
);
|
||||
const INTELLIGENCE_REPO = process.env.INTELLIGENCE_REPO
|
||||
? resolve(process.env.INTELLIGENCE_REPO)
|
||||
: DEFAULT_INTELLIGENCE_REPO;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const WRITE = args.includes("--write");
|
||||
const orgFlagIdx = args.indexOf("--org");
|
||||
const ORG_ID = orgFlagIdx !== -1 ? args[orgFlagIdx + 1] : "casa-de-erlang";
|
||||
|
||||
function die(msg) {
|
||||
console.error(`\n✗ mint-dev-license: ${msg}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// --- Preflight: the private signer source + the repo's tsx must be present ---
|
||||
const signerEntry = resolve(
|
||||
INTELLIGENCE_REPO,
|
||||
"libs/license-signing/src/index.ts",
|
||||
);
|
||||
if (!existsSync(signerEntry)) {
|
||||
die(
|
||||
`Intelligence signer not found at ${signerEntry}.\n` +
|
||||
` This dev-license path needs the private Intelligence source. Set INTELLIGENCE_REPO\n` +
|
||||
` to your Intelligence checkout, e.g. INTELLIGENCE_REPO=/path/to/Intelligence\n` +
|
||||
` NOT needed for MANAGED Intelligence — use a CopilotKit-issued license instead\n` +
|
||||
` (see .env.example).`,
|
||||
);
|
||||
}
|
||||
const tsxBin = resolve(INTELLIGENCE_REPO, "node_modules/.bin/tsx");
|
||||
if (!existsSync(tsxBin)) {
|
||||
die(
|
||||
`tsx not found at ${tsxBin}. Run 'pnpm install' in the Intelligence repo first.`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- Mint: run a throwaway signer inside the Intelligence repo context ---
|
||||
// The signer imports @cpki/license-catalog via that repo's tsconfig paths, so
|
||||
// the temp file must live inside the repo tree (relative import + repo cwd).
|
||||
const repoTmpDir = resolve(INTELLIGENCE_REPO, "tmp");
|
||||
mkdirSync(repoTmpDir, { recursive: true });
|
||||
const tmpSigner = resolve(
|
||||
repoTmpDir,
|
||||
`_mint-banking-license.${process.pid}.ts`,
|
||||
);
|
||||
|
||||
const signerSource = `
|
||||
import { generateKeyPair, generateKeyId, createLicensePayload, signLicense, getDefaultFeatures } from '../libs/license-signing/src/index.ts';
|
||||
const kp = generateKeyPair();
|
||||
const keyId = generateKeyId();
|
||||
const payload = createLicensePayload(
|
||||
{ organizationId: ${JSON.stringify(ORG_ID)}, organizationName: 'banking-demo', contactEmail: 'demo@northwind.example',
|
||||
tier: 'enterprise', planCode: 'enterprise', entitlementSource: 'enterprise_override', issuer: 'banking-demo-local',
|
||||
seatLimit: 0, features: { ...getDefaultFeatures('enterprise'), memory: true }, removeBranding: true,
|
||||
expiresAt: new Date('2099-01-01T00:00:00Z'), telemetryId: 'banking-demo-local' },
|
||||
keyId, 'lic_banking_demo_local',
|
||||
);
|
||||
console.log('TOKEN=' + signLicense(payload, kp.privateKey));
|
||||
console.log('BAKED=' + JSON.stringify({ [keyId]: kp.publicKey }));
|
||||
`;
|
||||
|
||||
let out;
|
||||
try {
|
||||
writeFileSync(tmpSigner, signerSource);
|
||||
out = execFileSync(tsxBin, [tmpSigner], {
|
||||
cwd: INTELLIGENCE_REPO,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
} catch (err) {
|
||||
die(
|
||||
`signing failed inside ${INTELLIGENCE_REPO}. See the error above.\n ${err?.message ?? err}`,
|
||||
);
|
||||
} finally {
|
||||
rmSync(tmpSigner, { force: true });
|
||||
}
|
||||
|
||||
const token = out.match(/^TOKEN=(.+)$/m)?.[1];
|
||||
const baked = out.match(/^BAKED=(.+)$/m)?.[1];
|
||||
if (!token || !baked) die(`could not parse signer output:\n${out}`);
|
||||
|
||||
const envPairs = {
|
||||
// main renamed the deployment-mode env and uses the underscore value.
|
||||
INTELLIGENCE_DEPLOYMENT_MODE: "self_hosted",
|
||||
COPILOTKIT_LICENSE_TOKEN: token,
|
||||
BAKED_LICENSE_KEYS_JSON: baked,
|
||||
};
|
||||
|
||||
if (!WRITE) {
|
||||
console.log(
|
||||
`# Self-hosted dev license (org: ${ORG_ID}). Paste into .env, or re-run with --write.`,
|
||||
);
|
||||
for (const [k, v] of Object.entries(envPairs)) console.log(`${k}=${v}`);
|
||||
console.log(
|
||||
`\n# MANAGED Intelligence does NOT use BAKED_LICENSE_KEYS_JSON — see .env.example.`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --- --write: upsert the keys into ./.env, preserving everything else ---
|
||||
const envPath = resolve(DEMO_ROOT, ".env");
|
||||
let envText = existsSync(envPath) ? readFileSync(envPath, "utf8") : "";
|
||||
for (const [k, v] of Object.entries(envPairs)) {
|
||||
const line = `${k}=${v}`;
|
||||
const re = new RegExp(`^${k}=.*$`, "m");
|
||||
if (re.test(envText)) {
|
||||
envText = envText.replace(re, line);
|
||||
} else {
|
||||
if (envText.length && !envText.endsWith("\n")) envText += "\n";
|
||||
envText += `${line}\n`;
|
||||
}
|
||||
}
|
||||
writeFileSync(envPath, envText);
|
||||
console.log(
|
||||
`✓ Wrote INTELLIGENCE_DEPLOYMENT_MODE, COPILOTKIT_LICENSE_TOKEN, BAKED_LICENSE_KEYS_JSON to ${envPath}`,
|
||||
);
|
||||
console.log(
|
||||
` (org: ${ORG_ID}; self-hosted dev license, features.memory=true)`,
|
||||
);
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Over-limit gate smoke test — the premise the self-learning "teach a workflow"
|
||||
* loop rests on.
|
||||
*
|
||||
* The narrated teach loop has the agent recall a saved procedure and apply it to
|
||||
* a *different* over-limit charge. That only clears the charge if the gate is
|
||||
* lifted *exclusively* by a finalized exception filed under a JUSTIFYING code
|
||||
* (see src/lib/store.ts `hasApprovedException` +
|
||||
* src/app/api/v1/policy-exception-codes.ts `isJustifying`). The Save card echoes
|
||||
* the demonstrated justifying code (EXC-BOARD-APPROVED) back to the agent as the
|
||||
* canonical procedure; if a non-justifying code also cleared the gate, or a
|
||||
* justifying one didn't, that procedure would be wrong. This guards that premise.
|
||||
*
|
||||
* Proves, against the running demo server, on one over-limit transaction:
|
||||
* 1. approve with NO exception → 422 OVER_POLICY_LIMIT (precondition)
|
||||
* 2. approve after a NON-justifying exception → 422 OVER_POLICY_LIMIT (does not unlock)
|
||||
* 3. approve after a JUSTIFYING exception → 201 (unlocks)
|
||||
*
|
||||
* The in-memory store mutates (step 3 approves the txn), so run this against a
|
||||
* freshly-started server, and restart the server before a live demo.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/over-limit-gate-smoke.mjs
|
||||
* Env (optional):
|
||||
* DEMO_URL default http://localhost:3000
|
||||
* GATE_TXN_ID default t-3 (an over-limit pending seed txn NOT used in the demo beats)
|
||||
*/
|
||||
|
||||
const DEMO_URL = process.env.DEMO_URL ?? "http://localhost:3000";
|
||||
const TXN_ID = process.env.GATE_TXN_ID ?? "t-3";
|
||||
|
||||
// EXC-BOARD-APPROVED must stay in sync with the justifying code the Save card's
|
||||
// CANONICAL_PROCEDURE echoes to the agent (src/app/page.tsx).
|
||||
const JUSTIFYING_CODE = "EXC-BOARD-APPROVED";
|
||||
const NON_JUSTIFYING_CODE = "EXC-WILL-REIMBURSE";
|
||||
|
||||
const results = [];
|
||||
function check(name, ok, detail) {
|
||||
results.push({ name, ok });
|
||||
console.log(`${ok ? "✓" : "✗"} ${name}${detail ? ` — ${detail}` : ""}`);
|
||||
return ok;
|
||||
}
|
||||
|
||||
async function approve(id) {
|
||||
const res = await fetch(`${DEMO_URL}/api/v1/transactions/${id}`, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ status: "approved" }),
|
||||
});
|
||||
const body = await res.json().catch(() => null);
|
||||
return { status: res.status, body };
|
||||
}
|
||||
|
||||
async function openException(transactionId, code) {
|
||||
const res = await fetch(`${DEMO_URL}/api/v1/exceptions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ transactionId, code }),
|
||||
});
|
||||
const body = await res.json().catch(() => null);
|
||||
if (!res.ok) throw new Error(`open ${code}: HTTP ${res.status}`);
|
||||
return body.id;
|
||||
}
|
||||
|
||||
async function finalize(exceptionId) {
|
||||
const res = await fetch(
|
||||
`${DEMO_URL}/api/v1/exceptions/${exceptionId}/finalize`,
|
||||
{ method: "POST", headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
if (!res.ok) throw new Error(`finalize ${exceptionId}: HTTP ${res.status}`);
|
||||
}
|
||||
|
||||
console.log(`over-limit gate smoke — ${DEMO_URL}, txn ${TXN_ID}\n`);
|
||||
|
||||
try {
|
||||
// 1. Precondition: the seed txn is over its policy limit and unapproved.
|
||||
const pre = await approve(TXN_ID);
|
||||
check(
|
||||
"1. approve with no exception is rejected",
|
||||
pre.status === 422 && pre.body?.error === "OVER_POLICY_LIMIT",
|
||||
`HTTP ${pre.status} ${pre.body?.error ?? ""}`,
|
||||
);
|
||||
|
||||
// 2. A non-justifying exception files but does NOT lift the gate.
|
||||
const nonJustId = await openException(TXN_ID, NON_JUSTIFYING_CODE);
|
||||
await finalize(nonJustId);
|
||||
const afterNonJust = await approve(TXN_ID);
|
||||
check(
|
||||
"2. non-justifying exception does not unlock approval",
|
||||
afterNonJust.status === 422 &&
|
||||
afterNonJust.body?.error === "OVER_POLICY_LIMIT",
|
||||
`HTTP ${afterNonJust.status} ${afterNonJust.body?.error ?? ""}`,
|
||||
);
|
||||
|
||||
// 3. A justifying exception lifts the gate — approval now succeeds.
|
||||
const justId = await openException(TXN_ID, JUSTIFYING_CODE);
|
||||
await finalize(justId);
|
||||
const afterJust = await approve(TXN_ID);
|
||||
check(
|
||||
"3. justifying exception unlocks approval",
|
||||
afterJust.status === 201,
|
||||
`HTTP ${afterJust.status}`,
|
||||
);
|
||||
} catch (error) {
|
||||
check("gate smoke", false, String(error).slice(0, 200));
|
||||
}
|
||||
|
||||
const failed = results.filter((r) => !r.ok);
|
||||
console.log(
|
||||
`\n${failed.length === 0 ? "PASS" : "FAIL"} — ${results.length - failed.length}/${results.length} checks passed`,
|
||||
);
|
||||
if (failed.length > 0) {
|
||||
console.log(
|
||||
"hint: run against a FRESHLY-started server (the in-memory store mutates; " +
|
||||
"a prior run leaves the txn approved). Confirm the seed txn is over-limit.",
|
||||
);
|
||||
}
|
||||
process.exit(failed.length === 0 ? 0 : 1);
|
||||
Reference in New Issue
Block a user