chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,52 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
|
||||
|
||||
describe("BoundedTtlCache", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("returns a live entry within its TTL", () => {
|
||||
vi.useFakeTimers();
|
||||
const cache = new BoundedTtlCache<string>(1_000, 100);
|
||||
cache.set("k", "v");
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(cache.get("k")).toBe("v");
|
||||
expect(cache.size).toBe(1);
|
||||
});
|
||||
|
||||
it("evicts an expired entry on read instead of letting it linger", () => {
|
||||
vi.useFakeTimers();
|
||||
const cache = new BoundedTtlCache<number>(1_000, 100);
|
||||
cache.set("a", 1);
|
||||
expect(cache.size).toBe(1);
|
||||
|
||||
vi.advanceTimersByTime(1_001);
|
||||
expect(cache.get("a")).toBeUndefined();
|
||||
// The previous bug left expired entries in the map until an at-capacity sweep;
|
||||
// they must now be removed on read.
|
||||
expect(cache.size).toBe(0);
|
||||
});
|
||||
|
||||
it("does not evict another entry when updating an existing key at capacity", () => {
|
||||
const cache = new BoundedTtlCache<number>(60_000, 2);
|
||||
cache.set("a", 1);
|
||||
cache.set("b", 2);
|
||||
// Updating an existing key doesn't grow the map, so it must not drop "b".
|
||||
cache.set("a", 11);
|
||||
expect(cache.get("a")).toBe(11);
|
||||
expect(cache.get("b")).toBe(2);
|
||||
expect(cache.size).toBe(2);
|
||||
});
|
||||
|
||||
it("drops the oldest entry when full of still-live entries", () => {
|
||||
const cache = new BoundedTtlCache<number>(60_000, 2);
|
||||
cache.set("a", 1);
|
||||
cache.set("b", 2);
|
||||
cache.set("c", 3); // over capacity, none expired -> evict oldest insertion (a)
|
||||
expect(cache.get("a")).toBeUndefined();
|
||||
expect(cache.get("b")).toBe(2);
|
||||
expect(cache.get("c")).toBe(3);
|
||||
expect(cache.size).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,388 @@
|
||||
import { describe, expect, vi } from "vitest";
|
||||
|
||||
// The runsRepository module graph imports `~/v3/runStore.server`, which imports `~/db.server`
|
||||
// at load. Stub it (the existing runsRepository.part*.test.ts / readthrough test do the same) — the
|
||||
// resolver under test is driven entirely through injected real containers, never the stubbed
|
||||
// module singletons.
|
||||
vi.mock("~/db.server", () => ({
|
||||
prisma: {},
|
||||
$replica: {},
|
||||
}));
|
||||
|
||||
import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers";
|
||||
import { PrismaClient } from "@trigger.dev/database";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { ClickHouseRunListResolver } from "~/services/realtime/clickHouseRunListResolver.server";
|
||||
import { setupClickhouseReplication } from "../utils/replicationUtils";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
|
||||
type SeedContext = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates the org/project/env parents on a single prisma client. TaskRun FKs require these to
|
||||
* exist, and this container doubles as the logical-replication source that feeds the
|
||||
* ClickHouse id-set, so all runs whose ids we expect from ClickHouse are seeded here.
|
||||
*/
|
||||
async function seedParents(prisma: PrismaClient, slug: string): Promise<SeedContext> {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `org-${slug}`, slug: `org-${slug}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `proj-${slug}`,
|
||||
slug: `proj-${slug}`,
|
||||
organizationId: organization.id,
|
||||
externalRef: `proj-${slug}`,
|
||||
},
|
||||
});
|
||||
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: `env-${slug}`,
|
||||
type: "DEVELOPMENT",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${slug}`,
|
||||
pkApiKey: `pk_dev_${slug}`,
|
||||
shortcode: `sc-${slug}`,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
organizationId: organization.id,
|
||||
projectId: project.id,
|
||||
environmentId: runtimeEnvironment.id,
|
||||
};
|
||||
}
|
||||
|
||||
/** A second environment in the same project — used to prove the CH filter excludes other envs. */
|
||||
async function seedSecondEnvironment(prisma: PrismaClient, ctx: SeedContext, slug: string) {
|
||||
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: `env-${slug}-2`,
|
||||
type: "PRODUCTION",
|
||||
projectId: ctx.projectId,
|
||||
organizationId: ctx.organizationId,
|
||||
apiKey: `tr_prod_${slug}`,
|
||||
pkApiKey: `pk_prod_${slug}`,
|
||||
shortcode: `sc-${slug}-2`,
|
||||
},
|
||||
});
|
||||
return runtimeEnvironment.id;
|
||||
}
|
||||
|
||||
async function createRun(
|
||||
prisma: PrismaClient,
|
||||
ctx: SeedContext & { environmentId?: string },
|
||||
run: { friendlyId: string; runTags?: string[]; createdAt?: Date }
|
||||
) {
|
||||
return prisma.taskRun.create({
|
||||
data: {
|
||||
friendlyId: run.friendlyId,
|
||||
taskIdentifier: "my-task",
|
||||
status: "PENDING",
|
||||
payload: JSON.stringify({ foo: run.friendlyId }),
|
||||
traceId: run.friendlyId,
|
||||
spanId: run.friendlyId,
|
||||
queue: "test",
|
||||
runTags: run.runTags ?? [],
|
||||
...(run.createdAt ? { createdAt: run.createdAt } : {}),
|
||||
runtimeEnvironmentId: ctx.environmentId,
|
||||
projectId: ctx.projectId,
|
||||
organizationId: ctx.organizationId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
engine: "V2",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a real prisma client so ONLY `taskRun.findMany` throws — every other member stays a real
|
||||
* handle. The resolver's id-set path (`listRunIds` -> `listRunRows`) performs no `taskRun.findMany`
|
||||
* and never calls the run-ops store, so this proxy must never trip. The CPRES-owned filter
|
||||
* resolution that DOES run for a `batchId` filter uses `batchTaskRun.findFirst`, which this proxy
|
||||
* leaves intact.
|
||||
*/
|
||||
function throwingTaskRunFindMany(prisma: PrismaClient): PrismaClient {
|
||||
return new Proxy(prisma, {
|
||||
get(target, prop) {
|
||||
if (prop === "taskRun") {
|
||||
return new Proxy((target as any).taskRun, {
|
||||
get(trTarget, trProp) {
|
||||
if (trProp === "findMany") {
|
||||
return async () => {
|
||||
throw new Error(
|
||||
"taskRun.findMany must not be invoked on the realtime id-set path (a hydrate leaked in)"
|
||||
);
|
||||
};
|
||||
}
|
||||
return (trTarget as any)[trProp];
|
||||
},
|
||||
});
|
||||
}
|
||||
return (target as any)[prop];
|
||||
},
|
||||
}) as unknown as PrismaClient;
|
||||
}
|
||||
|
||||
describe("ClickHouseRunListResolver (realtime run-list id-set, split-neutral)", () => {
|
||||
// resolves the CH id-set with NO TaskRun PG hydrate.
|
||||
replicationContainerTest(
|
||||
"resolves the ClickHouse id-set for run-ops rows without ever reading TaskRun in Postgres",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
const ctx = await seedParents(prisma, "idset");
|
||||
|
||||
const runA = await createRun(prisma, ctx, { friendlyId: "run_idsetA" });
|
||||
const runB = await createRun(prisma, ctx, { friendlyId: "run_idsetB" });
|
||||
const runC = await createRun(prisma, ctx, { friendlyId: "run_idsetC" });
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
// ONLY taskRun.findMany throws; the rest of the client is real so the resolver can run.
|
||||
const resolver = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma: throwingTaskRunFindMany(prisma),
|
||||
});
|
||||
|
||||
const runIds = await resolver.resolveMatchingRunIds({
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
// Asserting as a set: equal createdAt makes the CH (created_at, run_id) DESC tie-break the
|
||||
// only ordering signal. The throwing proxy never tripped -> no TaskRun hydrate on this path.
|
||||
expect([...runIds].sort()).toEqual([runA.id, runB.id, runC.id].sort());
|
||||
}
|
||||
);
|
||||
|
||||
// CH filter is split-neutral (ids independent of PG residency).
|
||||
replicationContainerTest(
|
||||
"returns the same id-set regardless of which Postgres the rows are hydrated from (CH-only path)",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
// A second, unrelated NEW client carrying NO rows. The id-set path never touches it;
|
||||
// pointing the resolver at it must not change the result, proving the ids come from CH only.
|
||||
const { url: newUrl } = await createPostgresContainer(network, {
|
||||
imageTag: "docker.io/postgres:17",
|
||||
});
|
||||
const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } });
|
||||
|
||||
try {
|
||||
const ctx = await seedParents(prisma, "neutral");
|
||||
const runA = await createRun(prisma, ctx, { friendlyId: "run_neutralA" });
|
||||
const runB = await createRun(prisma, ctx, { friendlyId: "run_neutralB" });
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
const filter = {
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
limit: 10,
|
||||
};
|
||||
|
||||
// "single-DB" wiring: resolver's prisma is the replication-source DB (where rows live).
|
||||
const singleDb = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma,
|
||||
});
|
||||
// "split" wiring: resolver's prisma is the empty NEW DB. If the id-set path read TaskRun
|
||||
// from this handle the result would differ; it must not.
|
||||
const split = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma: prismaNew,
|
||||
});
|
||||
|
||||
const idsSingleDb = await singleDb.resolveMatchingRunIds(filter);
|
||||
const idsSplit = await split.resolveMatchingRunIds(filter);
|
||||
|
||||
expect(idsSplit).toEqual(idsSingleDb);
|
||||
expect([...idsSingleDb].sort()).toEqual([runA.id, runB.id].sort());
|
||||
} finally {
|
||||
await prismaNew.$disconnect();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// single-DB passthrough; no legacy/known-migrated probe on this path.
|
||||
replicationContainerTest(
|
||||
"single-DB passthrough returns the CH id-set and never hydrates TaskRun",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
const ctx = await seedParents(prisma, "passthrough");
|
||||
const run = await createRun(prisma, ctx, { friendlyId: "run_passthrough" });
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
const resolver = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma: throwingTaskRunFindMany(prisma),
|
||||
});
|
||||
|
||||
const runIds = await resolver.resolveMatchingRunIds({
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
expect(runIds).toEqual([run.id]);
|
||||
}
|
||||
);
|
||||
|
||||
// a far-future straggler's id surfaces from the CH id-set.
|
||||
replicationContainerTest(
|
||||
"surfaces a far-future delayed run's id from the CH id-set",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
const ctx = await seedParents(prisma, "straggler");
|
||||
|
||||
const now = new Date();
|
||||
const near = await createRun(prisma, ctx, { friendlyId: "run_near", createdAt: now });
|
||||
// The migrated-by-sweep case: CH is residency-agnostic, so the id surfaces once indexed
|
||||
// regardless of which DB holds the row.
|
||||
const farFuture = new Date(now.getTime() + 365 * 24 * 60 * 60 * 1000);
|
||||
const straggler = await createRun(prisma, ctx, {
|
||||
friendlyId: "run_straggler",
|
||||
createdAt: farFuture,
|
||||
});
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
const resolver = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma,
|
||||
});
|
||||
|
||||
const runIds = await resolver.resolveMatchingRunIds({
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
expect(runIds).toContain(straggler.id);
|
||||
expect(runIds).toContain(near.id);
|
||||
// (created_at, run_id) DESC ordering -> the far-future straggler sorts ahead of the near run.
|
||||
expect(runIds.indexOf(straggler.id)).toBeLessThan(runIds.indexOf(near.id));
|
||||
}
|
||||
);
|
||||
|
||||
// tag match is contains-ALL (tagsMatch: "all" -> hasAll), authoritative.
|
||||
// The sibling runReader.server.ts JSDoc still calls RunListFilter.tags "Contains-ANY"; that is
|
||||
// stale. The resolver passes tagsMatch: "all" and the live CH repo maps
|
||||
// it to hasAll, so contains-ALL is the real behavior — assert that, not the JSDoc.
|
||||
replicationContainerTest(
|
||||
"tag filter is contains-ALL: only runs carrying every requested tag are returned",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
const ctx = await seedParents(prisma, "tags");
|
||||
|
||||
// Has BOTH requested tags -> matches contains-ALL.
|
||||
const both = await createRun(prisma, ctx, {
|
||||
friendlyId: "run_bothTags",
|
||||
runTags: ["alpha", "beta"],
|
||||
});
|
||||
// Has only one of the requested tags -> excluded under contains-ALL (would match contains-ANY).
|
||||
await createRun(prisma, ctx, { friendlyId: "run_oneTag", runTags: ["alpha"] });
|
||||
// Has neither -> excluded.
|
||||
await createRun(prisma, ctx, { friendlyId: "run_otherTag", runTags: ["gamma"] });
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
const resolver = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma,
|
||||
});
|
||||
|
||||
const runIds = await resolver.resolveMatchingRunIds({
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
tags: ["alpha", "beta"],
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
// contains-ALL: only the run with BOTH tags. (contains-ANY would also return run_oneTag.)
|
||||
expect(runIds).toEqual([both.id]);
|
||||
}
|
||||
);
|
||||
|
||||
// environment scoping: the CH filter excludes other environments.
|
||||
// Doubles as a structural proof that an accidental hydrate would NOT change the id-set: rows on a
|
||||
// different env are not returned because CH filters by environment_id, not because PG was read.
|
||||
replicationContainerTest(
|
||||
"scopes the id-set to the filtered environment (other-env rows are excluded by the CH filter)",
|
||||
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
|
||||
const { clickhouse } = await setupClickhouseReplication({
|
||||
prisma,
|
||||
databaseUrl: postgresContainer.getConnectionUri(),
|
||||
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
|
||||
redisOptions,
|
||||
});
|
||||
|
||||
const ctx = await seedParents(prisma, "envscope");
|
||||
const otherEnvId = await seedSecondEnvironment(prisma, ctx, "envscope");
|
||||
|
||||
const inEnv = await createRun(prisma, ctx, { friendlyId: "run_inEnv" });
|
||||
await createRun(
|
||||
prisma,
|
||||
{ ...ctx, environmentId: otherEnvId },
|
||||
{ friendlyId: "run_otherEnv" }
|
||||
);
|
||||
|
||||
await setTimeout(1500);
|
||||
|
||||
const resolver = new ClickHouseRunListResolver({
|
||||
getClickhouse: async () => clickhouse,
|
||||
prisma: throwingTaskRunFindMany(prisma),
|
||||
});
|
||||
|
||||
const runIds = await resolver.resolveMatchingRunIds({
|
||||
organizationId: ctx.organizationId,
|
||||
projectId: ctx.projectId,
|
||||
environmentId: ctx.environmentId,
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
expect(runIds).toEqual([inEnv.id]);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,304 @@
|
||||
import { SubscribeRunRawShape } from "@trigger.dev/core/v3/schemas";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildElectricSchemaHeader,
|
||||
buildRowsBody,
|
||||
buildSnapshotBody,
|
||||
buildUpdateBody,
|
||||
buildUpToDateBody,
|
||||
encodeOffset,
|
||||
parseOffsetUpdatedAtMs,
|
||||
type RealtimeRunRow,
|
||||
rewriteBodyForLegacyApiVersion,
|
||||
serializeRunRow,
|
||||
} from "~/services/realtime/electricStreamProtocol.server";
|
||||
|
||||
function sampleRow(overrides: Partial<RealtimeRunRow> = {}): RealtimeRunRow {
|
||||
return {
|
||||
id: "run_abc123",
|
||||
taskIdentifier: "my-task",
|
||||
createdAt: new Date("2026-06-06T10:00:00.000Z"),
|
||||
updatedAt: new Date("2026-06-06T10:05:30.123Z"),
|
||||
startedAt: new Date("2026-06-06T10:01:00.000Z"),
|
||||
delayUntil: null,
|
||||
queuedAt: new Date("2026-06-06T10:00:30.000Z"),
|
||||
expiredAt: null,
|
||||
completedAt: null,
|
||||
friendlyId: "run_friendly_abc",
|
||||
number: 42,
|
||||
isTest: true,
|
||||
status: "EXECUTING",
|
||||
usageDurationMs: 1234,
|
||||
costInCents: 0.55,
|
||||
baseCostInCents: 0.25,
|
||||
ttl: "1h",
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
metadata: '{"step":1}',
|
||||
metadataType: "application/json",
|
||||
output: null,
|
||||
outputType: "application/json",
|
||||
runTags: ["user:123", "env:prod"],
|
||||
error: null,
|
||||
realtimeStreams: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Faithful re-implementation of the @electric-sql/client value parser rules
|
||||
* (defaultParser + pgArrayParser), so we can decode our wire `value` object the
|
||||
* same way the deployed client would, then validate against the real SDK schema.
|
||||
* Source: @electric-sql/client@1.0.14 src/parser.ts.
|
||||
*/
|
||||
function electricParse(
|
||||
value: Record<string, string | null>,
|
||||
schema: Record<string, { type: string; dims?: number }>
|
||||
): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, raw] of Object.entries(value)) {
|
||||
if (raw === null) {
|
||||
out[key] = null;
|
||||
continue;
|
||||
}
|
||||
const info = schema[key];
|
||||
if (!info) {
|
||||
out[key] = raw;
|
||||
continue;
|
||||
}
|
||||
if (info.dims && info.dims > 0) {
|
||||
out[key] = parsePgTextArray(raw);
|
||||
continue;
|
||||
}
|
||||
switch (info.type) {
|
||||
case "bool":
|
||||
out[key] = raw === "t" || raw === "true";
|
||||
break;
|
||||
case "int8":
|
||||
out[key] = BigInt(raw);
|
||||
break;
|
||||
case "int2":
|
||||
case "int4":
|
||||
case "float4":
|
||||
case "float8":
|
||||
out[key] = Number(raw);
|
||||
break;
|
||||
case "json":
|
||||
case "jsonb":
|
||||
out[key] = JSON.parse(raw);
|
||||
break;
|
||||
default:
|
||||
out[key] = raw; // text/timestamp pass through as strings
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parsePgTextArray(literal: string): string[] {
|
||||
if (literal === "{}") {
|
||||
return [];
|
||||
}
|
||||
const inner = literal.slice(1, -1);
|
||||
const result: string[] = [];
|
||||
let i = 0;
|
||||
while (i < inner.length) {
|
||||
if (inner[i] === '"') {
|
||||
i++;
|
||||
let s = "";
|
||||
while (i < inner.length && inner[i] !== '"') {
|
||||
if (inner[i] === "\\") {
|
||||
i++;
|
||||
}
|
||||
s += inner[i];
|
||||
i++;
|
||||
}
|
||||
result.push(s);
|
||||
i++; // closing quote
|
||||
if (inner[i] === ",") i++;
|
||||
} else {
|
||||
let s = "";
|
||||
while (i < inner.length && inner[i] !== ",") {
|
||||
s += inner[i];
|
||||
i++;
|
||||
}
|
||||
result.push(s);
|
||||
if (inner[i] === ",") i++;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("electricStreamProtocol serializer", () => {
|
||||
it("encodes each Postgres type the way the Electric client expects", () => {
|
||||
const value = serializeRunRow(sampleRow());
|
||||
|
||||
// text: passed through as-is
|
||||
expect(value.id).toBe("run_abc123");
|
||||
expect(value.status).toBe("EXECUTING");
|
||||
expect(value.payload).toBe('{"hello":"world"}');
|
||||
|
||||
// int/float: stringified
|
||||
expect(value.number).toBe("42");
|
||||
expect(value.usageDurationMs).toBe("1234");
|
||||
expect(value.costInCents).toBe("0.55");
|
||||
|
||||
// bool: postgres "t"/"f"
|
||||
expect(value.isTest).toBe("t");
|
||||
|
||||
// timestamp: ISO without trailing Z (the SDK appends Z before parsing)
|
||||
expect(value.updatedAt).toBe("2026-06-06T10:05:30.123");
|
||||
expect(value.createdAt).toBe("2026-06-06T10:00:00.000");
|
||||
|
||||
// nullable timestamp: null stays null
|
||||
expect(value.delayUntil).toBeNull();
|
||||
expect(value.completedAt).toBeNull();
|
||||
|
||||
// text[]: quoted pg array literal; empty realtimeStreams (@default([])) => {}
|
||||
expect(value.runTags).toBe('{"user:123","env:prod"}');
|
||||
expect(value.realtimeStreams).toBe("{}");
|
||||
|
||||
// jsonb: null stays null
|
||||
expect(value.error).toBeNull();
|
||||
});
|
||||
|
||||
it("encodes an empty no-default array column (runTags) as null, matching Electric", () => {
|
||||
// runTags has no Postgres default, so an empty value is stored as SQL NULL and
|
||||
// Electric emits `null` (not `{}`). realtimeStreams has @default([]), so its
|
||||
// empty value is `{}`. Prisma hands us `[]` for both; we re-derive the wire form.
|
||||
const value = serializeRunRow(sampleRow({ runTags: [], realtimeStreams: [] }));
|
||||
expect(value.runTags).toBeNull();
|
||||
expect(value.realtimeStreams).toBe("{}");
|
||||
});
|
||||
|
||||
it("encodes jsonb error as a JSON string", () => {
|
||||
const value = serializeRunRow(sampleRow({ error: { type: "STRING_ERROR", raw: "boom" } }));
|
||||
expect(value.error).toBe('{"type":"STRING_ERROR","raw":"boom"}');
|
||||
});
|
||||
|
||||
it("round-trips through the client parser into a valid SubscribeRunRawShape", () => {
|
||||
const row = sampleRow({ error: { type: "STRING_ERROR", raw: "boom" } });
|
||||
const value = serializeRunRow(row);
|
||||
const schema = JSON.parse(buildElectricSchemaHeader());
|
||||
|
||||
const decoded = electricParse(value, schema);
|
||||
const parsed = SubscribeRunRawShape.parse(decoded);
|
||||
|
||||
expect(parsed.id).toBe("run_abc123");
|
||||
expect(parsed.friendlyId).toBe("run_friendly_abc");
|
||||
expect(parsed.status).toBe("EXECUTING");
|
||||
expect(parsed.number).toBe(42);
|
||||
expect(parsed.isTest).toBe(true);
|
||||
expect(parsed.usageDurationMs).toBe(1234);
|
||||
expect(parsed.costInCents).toBeCloseTo(0.55);
|
||||
expect(parsed.runTags).toEqual(["user:123", "env:prod"]);
|
||||
expect(parsed.realtimeStreams).toEqual([]);
|
||||
// RawShapeDate appends "Z" and coerces to a Date equal to the source instant.
|
||||
expect(parsed.createdAt.toISOString()).toBe("2026-06-06T10:00:00.000Z");
|
||||
expect(parsed.updatedAt.toISOString()).toBe("2026-06-06T10:05:30.123Z");
|
||||
expect(parsed.startedAt?.toISOString()).toBe("2026-06-06T10:01:00.000Z");
|
||||
expect(parsed.delayUntil ?? null).toBeNull();
|
||||
expect(parsed.error).toEqual({ type: "STRING_ERROR", raw: "boom" });
|
||||
});
|
||||
|
||||
it("honors skipColumns (but never the reserved columns)", () => {
|
||||
const value = serializeRunRow(sampleRow(), ["payload", "output", "id", "status"]);
|
||||
expect(value.payload).toBeUndefined();
|
||||
expect(value.output).toBeUndefined();
|
||||
// reserved columns can't be skipped
|
||||
expect(value.id).toBe("run_abc123");
|
||||
expect(value.status).toBe("EXECUTING");
|
||||
|
||||
const schema = JSON.parse(buildElectricSchemaHeader(["payload"]));
|
||||
expect(schema.payload).toBeUndefined();
|
||||
expect(schema.status).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("electricStreamProtocol message bodies", () => {
|
||||
it("emits insert + up-to-date for an initial snapshot", () => {
|
||||
const messages = JSON.parse(buildSnapshotBody(sampleRow()));
|
||||
expect(messages).toHaveLength(2);
|
||||
expect(messages[0].headers.operation).toBe("insert");
|
||||
expect(messages[0].key).toBe('"public"."TaskRun"/"run_abc123"');
|
||||
expect(messages[0].value.status).toBe("EXECUTING");
|
||||
expect(messages[1].headers.control).toBe("up-to-date");
|
||||
});
|
||||
|
||||
it("emits a bare up-to-date for an empty (missing) run snapshot", () => {
|
||||
const messages = JSON.parse(buildSnapshotBody(null));
|
||||
expect(messages).toHaveLength(1);
|
||||
expect(messages[0].headers.control).toBe("up-to-date");
|
||||
});
|
||||
|
||||
it("emits update + up-to-date for a live change", () => {
|
||||
const messages = JSON.parse(buildUpdateBody(sampleRow()));
|
||||
expect(messages[0].headers.operation).toBe("update");
|
||||
expect(messages[1].headers.control).toBe("up-to-date");
|
||||
});
|
||||
|
||||
it("emits a bare up-to-date when nothing advanced", () => {
|
||||
const messages = JSON.parse(buildUpToDateBody());
|
||||
expect(messages).toEqual([{ headers: { control: "up-to-date" } }]);
|
||||
});
|
||||
|
||||
it("uses the same merge key across insert and update so the client merges by row", () => {
|
||||
const insert = JSON.parse(buildSnapshotBody(sampleRow()))[0];
|
||||
const update = JSON.parse(buildUpdateBody(sampleRow()))[0];
|
||||
expect(insert.key).toBe(update.key);
|
||||
});
|
||||
});
|
||||
|
||||
describe("electricStreamProtocol multi-row (tag-list) bodies", () => {
|
||||
it("emits one change message per row with per-row operation, then up-to-date", () => {
|
||||
const a = sampleRow({ id: "run_a" });
|
||||
const b = sampleRow({ id: "run_b", status: "QUEUED" });
|
||||
const messages = JSON.parse(
|
||||
buildRowsBody([
|
||||
{ row: a, operation: "insert" },
|
||||
{ row: b, operation: "update" },
|
||||
])
|
||||
);
|
||||
expect(messages).toHaveLength(3);
|
||||
expect(messages[0].headers.operation).toBe("insert");
|
||||
expect(messages[0].key).toBe('"public"."TaskRun"/"run_a"');
|
||||
expect(messages[1].headers.operation).toBe("update");
|
||||
expect(messages[1].key).toBe('"public"."TaskRun"/"run_b"');
|
||||
expect(messages[1].value.status).toBe("QUEUED");
|
||||
expect(messages[2].headers.control).toBe("up-to-date");
|
||||
});
|
||||
|
||||
it("emits a bare up-to-date for an empty change set", () => {
|
||||
const messages = JSON.parse(buildRowsBody([]));
|
||||
expect(messages).toEqual([{ headers: { control: "up-to-date" } }]);
|
||||
});
|
||||
|
||||
it("honors skipColumns across all rows", () => {
|
||||
const messages = JSON.parse(
|
||||
buildRowsBody([{ row: sampleRow(), operation: "insert" }], ["payload"])
|
||||
);
|
||||
expect(messages[0].value.payload).toBeUndefined();
|
||||
expect(messages[0].value.status).toBe("EXECUTING");
|
||||
});
|
||||
});
|
||||
|
||||
describe("electricStreamProtocol tokens + legacy rewrite", () => {
|
||||
it("encodes and parses the offset updatedAt segment", () => {
|
||||
const offset = encodeOffset(1717667130123, 7);
|
||||
expect(offset).toBe("1717667130123_7");
|
||||
expect(parseOffsetUpdatedAtMs(offset)).toBe(1717667130123);
|
||||
});
|
||||
|
||||
it("treats the initial offset (-1) and garbage as zero", () => {
|
||||
expect(parseOffsetUpdatedAtMs("-1")).toBe(0);
|
||||
expect(parseOffsetUpdatedAtMs(null)).toBe(0);
|
||||
expect(parseOffsetUpdatedAtMs("nonsense")).toBe(0);
|
||||
});
|
||||
|
||||
it("rewrites DEQUEUED to EXECUTING for legacy API versions", () => {
|
||||
const body = buildUpdateBody(sampleRow({ status: "DEQUEUED" }));
|
||||
expect(body).toContain('"status":"DEQUEUED"');
|
||||
const rewritten = rewriteBodyForLegacyApiVersion(body);
|
||||
expect(rewritten).not.toContain('"status":"DEQUEUED"');
|
||||
expect(rewritten).toContain('"status":"EXECUTING"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,553 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
EnvChangeRouter,
|
||||
type EnvChangeSource,
|
||||
type RowHydrator,
|
||||
} from "~/services/realtime/envChangeRouter.server";
|
||||
import { type ChangeRecord } from "~/services/realtime/runChangeNotifier.server";
|
||||
import { type RealtimeRunRow } from "~/services/realtime/electricStreamProtocol.server";
|
||||
|
||||
const FLOOR_MS = Date.UTC(2026, 5, 7, 12, 0, 0);
|
||||
|
||||
function row(
|
||||
id: string,
|
||||
opts: { tags?: string[]; createdAtMs?: number; updatedAtMs?: number } = {}
|
||||
): RealtimeRunRow {
|
||||
return {
|
||||
id,
|
||||
runTags: opts.tags ?? [],
|
||||
createdAt: new Date(opts.createdAtMs ?? FLOOR_MS + 1_000),
|
||||
updatedAt: new Date(opts.updatedAtMs ?? FLOOR_MS + 5_000),
|
||||
} as unknown as RealtimeRunRow;
|
||||
}
|
||||
|
||||
function record(runId: string, extra: Partial<ChangeRecord> = {}): ChangeRecord {
|
||||
return { v: 1, runId, envId: "env_1", ...extra };
|
||||
}
|
||||
|
||||
/** A controllable EnvChangeSource: tests push batches to the env's listener. */
|
||||
function fakeSource() {
|
||||
const listeners = new Map<string, Set<(records: ChangeRecord[]) => void>>();
|
||||
const source: EnvChangeSource = {
|
||||
subscribeToEnv(envId, onBatch) {
|
||||
let set = listeners.get(envId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(envId, set);
|
||||
}
|
||||
set.add(onBatch);
|
||||
return () => {
|
||||
listeners.get(envId)?.delete(onBatch);
|
||||
};
|
||||
},
|
||||
};
|
||||
return {
|
||||
source,
|
||||
push(envId: string, records: ChangeRecord[]) {
|
||||
for (const l of listeners.get(envId) ?? []) l(records);
|
||||
},
|
||||
isSubscribed(envId: string) {
|
||||
return (listeners.get(envId)?.size ?? 0) > 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeRouter(
|
||||
rowsById: Map<string, RealtimeRunRow> = new Map(),
|
||||
options: Record<string, unknown> = {}
|
||||
) {
|
||||
const src = fakeSource();
|
||||
const hydrateSpy = vi.fn<RowHydrator["hydrateByIds"]>(async (_env, ids) =>
|
||||
ids.map((id) => rowsById.get(id)).filter((r): r is RealtimeRunRow => Boolean(r))
|
||||
);
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
...options,
|
||||
});
|
||||
return { router, src, hydrateSpy };
|
||||
}
|
||||
|
||||
describe("EnvChangeRouter", () => {
|
||||
it("routes a tag match to the feed (hydrated + serialized) and ignores non-matches", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 1_000);
|
||||
|
||||
// A non-matching tag is dropped (no wake); a matching tag wakes with the hydrated row.
|
||||
src.push("env_1", [record("rX", { tags: ["b"] }), record("r1", { tags: ["a"] })]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
expect(result.rows[0].value.id).toBe("r1"); // serialized wire value
|
||||
expect(hydrateSpy).toHaveBeenCalledWith("env_1", ["r1"], []);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("wakes an unfiltered tag feed (no tags) for every full record, live and via replay", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
|
||||
// Live path: a full record (tags defined) must reach the zero-filter feed even
|
||||
// though it can never appear in the byTag index.
|
||||
const reg = router.register("env_1", { kind: "tag", tags: [] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 1_000);
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
const live = await wait;
|
||||
expect(live.reason).toBe("notify");
|
||||
expect(live.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
reg.close();
|
||||
|
||||
// Replay path: the buffered record matches an unfiltered feed registered after the push.
|
||||
const late = router.register("env_1", { kind: "tag", tags: [] }, [], {
|
||||
replaySinceMs: Date.now() - 1_000,
|
||||
});
|
||||
const replayed = await late.waitForMatch(undefined, 1_000);
|
||||
expect(replayed.reason).toBe("notify");
|
||||
expect(replayed.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
late.close();
|
||||
});
|
||||
|
||||
it("batch-hydrates ONCE and shares the serialized value across feeds matching the same run", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows);
|
||||
const regs = [
|
||||
router.register("env_1", { kind: "tag", tags: ["a"] }, []),
|
||||
router.register("env_1", { kind: "tag", tags: ["a"] }, []),
|
||||
];
|
||||
const waits = regs.map((r) => r.waitForMatch(undefined, 1_000));
|
||||
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
const results = await Promise.all(waits);
|
||||
|
||||
// One hydrate for the whole tick (same column set), shared by both feeds...
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
// ...and the same serialized value object is reused (serialize-once).
|
||||
expect(results[0].rows[0].value).toBe(results[1].rows[0].value);
|
||||
regs.forEach((r) => r.close());
|
||||
});
|
||||
|
||||
it("a hydrate failure doesn't reject out of the source callback; the feed times out", async () => {
|
||||
const src = fakeSource();
|
||||
const hydrateSpy = vi.fn<RowHydrator["hydrateByIds"]>(async () => {
|
||||
throw new Error("replica down");
|
||||
});
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "run", runId: "r1" }, []);
|
||||
const wait = reg.waitForMatch(undefined, 50);
|
||||
|
||||
// Would be an unhandled rejection (process exit) if #onBatch's promise were unguarded.
|
||||
src.push("env_1", [record("r1")]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("timeout");
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("routes a run feed by exact runId", async () => {
|
||||
const rows = new Map([["r1", row("r1")]]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "run", runId: "r1" }, []);
|
||||
const wait = reg.waitForMatch(undefined, 1_000);
|
||||
src.push("env_1", [record("r2"), record("r1")]);
|
||||
const result = await wait;
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("routes a batch feed by batchId", async () => {
|
||||
const rows = new Map([["r1", row("r1")]]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "batch", batchId: "batch_1" }, []);
|
||||
const wait = reg.waitForMatch(undefined, 1_000);
|
||||
src.push("env_1", [record("rX", { batchId: "other" }), record("r1", { batchId: "batch_1" })]);
|
||||
const result = await wait;
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("multi-tag feeds require ALL tags on the row (Electric contains-all semantics)", async () => {
|
||||
const rows = new Map([
|
||||
["r_both", row("r_both", { tags: ["a", "b", "c"] })],
|
||||
["r_one", row("r_one", { tags: ["a"] })],
|
||||
]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a", "b"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 1_000);
|
||||
|
||||
// r_one shares a tag (routes as a candidate via the index) but lacks "b" — must be
|
||||
// culled by the authoritative row check. r_both carries both and wakes the feed.
|
||||
src.push("env_1", [
|
||||
record("r_one", { tags: ["a"] }),
|
||||
record("r_both", { tags: ["a", "b", "c"] }),
|
||||
]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r_both"]);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("drops a tag match created before the feed's createdAt floor", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"], createdAtMs: FLOOR_MS - 10_000 })]]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
const reg = router.register(
|
||||
"env_1",
|
||||
{ kind: "tag", tags: ["a"], createdAtFloorMs: FLOOR_MS },
|
||||
[]
|
||||
);
|
||||
let settled = false;
|
||||
const wait = reg.waitForMatch(undefined, 60).then((r) => {
|
||||
settled = true;
|
||||
return r;
|
||||
});
|
||||
src.push("env_1", [record("r1", { tags: ["a"], createdAtMs: FLOOR_MS - 10_000 })]);
|
||||
// Hydrated but out-of-window -> not woken; falls through to the timeout.
|
||||
const result = await wait;
|
||||
expect(settled).toBe(true);
|
||||
expect(result.reason).toBe("timeout");
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("classifies a partial record (no tags) by hydrating and re-checking the row's tags", async () => {
|
||||
// Partial record routes to all tag feeds as candidates; the authoritative row decides.
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src } = makeRouter(rows);
|
||||
const match = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const noMatch = router.register("env_1", { kind: "tag", tags: ["z"] }, []);
|
||||
const matchWait = match.waitForMatch(undefined, 1_000);
|
||||
let noMatchSettled = false;
|
||||
const noMatchWait = noMatch.waitForMatch(undefined, 80).then((r) => {
|
||||
noMatchSettled = true;
|
||||
return r;
|
||||
});
|
||||
|
||||
src.push("env_1", [record("r1", { tags: undefined })]); // partial: tags absent
|
||||
|
||||
expect((await matchWait).rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
expect((await noMatchWait).reason).toBe("timeout"); // row tags ["a"] don't intersect ["z"]
|
||||
expect(noMatchSettled).toBe(true);
|
||||
match.close();
|
||||
noMatch.close();
|
||||
});
|
||||
|
||||
it("times out and aborts cleanly", async () => {
|
||||
const { router, src } = makeRouter(new Map(), { unsubscribeLingerMs: 0 });
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
expect((await reg.waitForMatch(undefined, 30)).reason).toBe("timeout");
|
||||
|
||||
const controller = new AbortController();
|
||||
const wait = reg.waitForMatch(controller.signal, 5_000);
|
||||
controller.abort();
|
||||
expect((await wait).reason).toBe("abort");
|
||||
reg.close();
|
||||
expect(src.isSubscribed("env_1")).toBe(false); // linger disabled: last feed left -> unsubscribed
|
||||
});
|
||||
|
||||
it("buffers a record that arrives between polls and replays it on the next arm", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
// Not waiting yet: the push can't wake anything, but it lands in the env buffer.
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
expect(hydrateSpy).not.toHaveBeenCalled();
|
||||
|
||||
const result = await reg.waitForMatch(undefined, 1_000);
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("does not redeliver a replayed record on a later arm", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows);
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
expect((await reg.waitForMatch(undefined, 1_000)).reason).toBe("notify");
|
||||
|
||||
// Same buffered record must not fire again; the wait falls through to its timeout.
|
||||
expect((await reg.waitForMatch(undefined, 50)).reason).toBe("timeout");
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("lingers the env subscription after the last feed closes and replays the gap", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows, { unsubscribeLingerMs: 60 });
|
||||
const reg1 = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
reg1.close();
|
||||
expect(src.isSubscribed("env_1")).toBe(true); // lingering
|
||||
|
||||
// The inter-poll gap: a change arrives while no feed is registered.
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
|
||||
const reg2 = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const result = await reg2.waitForMatch(undefined, 1_000);
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
reg2.close();
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
expect(src.isSubscribed("env_1")).toBe(false); // linger expired -> unsubscribed
|
||||
});
|
||||
|
||||
it("reports gapCovered=false on a fresh env subscription and true once it ages past the window", async () => {
|
||||
const { router } = makeRouter(new Map(), { replayWindowMs: 50 });
|
||||
const reg1 = router.register("env_1", { kind: "run", runId: "r1" }, []);
|
||||
expect(reg1.gapCovered).toBe(false);
|
||||
|
||||
await new Promise((r) => setTimeout(r, 70));
|
||||
const reg2 = router.register("env_1", { kind: "run", runId: "r2" }, []);
|
||||
expect(reg2.gapCovered).toBe(true);
|
||||
reg1.close();
|
||||
reg2.close();
|
||||
});
|
||||
|
||||
it("honors the caller's replaySinceMs so a new poll doesn't rewind into delivered records", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const { router, src, hydrateSpy } = makeRouter(rows);
|
||||
const anchor = router.register("env_1", { kind: "tag", tags: ["a"] }, []); // keeps the env subscribed
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]);
|
||||
const afterPush = Date.now();
|
||||
|
||||
// A connection whose last response left after the push: nothing to replay.
|
||||
const caughtUp = router.register("env_1", { kind: "tag", tags: ["a"] }, [], {
|
||||
replaySinceMs: afterPush,
|
||||
});
|
||||
expect(caughtUp.gapCovered).toBe(true); // env subscribed since before its gap began
|
||||
expect((await caughtUp.waitForMatch(undefined, 50)).reason).toBe("timeout");
|
||||
expect(hydrateSpy).not.toHaveBeenCalled();
|
||||
|
||||
// A connection whose gap started before the push: the record replays.
|
||||
const behind = router.register("env_1", { kind: "tag", tags: ["a"] }, [], {
|
||||
replaySinceMs: afterPush - 1_000,
|
||||
});
|
||||
const result = await behind.waitForMatch(undefined, 1_000);
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows.map((m) => m.row.id)).toEqual(["r1"]);
|
||||
|
||||
anchor.close();
|
||||
caughtUp.close();
|
||||
behind.close();
|
||||
});
|
||||
|
||||
it("caps the replay buffer to the newest records per env", async () => {
|
||||
const rows = new Map([
|
||||
["r1", row("r1")],
|
||||
["r2", row("r2")],
|
||||
["r3", row("r3")],
|
||||
]);
|
||||
const evictions: string[] = [];
|
||||
const { router, src, hydrateSpy } = makeRouter(rows, {
|
||||
replayMaxRunsPerEnv: 2,
|
||||
onReplayEviction: (reason: string) => evictions.push(reason),
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "batch", batchId: "batch_1" }, []);
|
||||
src.push("env_1", [
|
||||
record("r1", { batchId: "batch_1" }),
|
||||
record("r2", { batchId: "batch_1" }),
|
||||
record("r3", { batchId: "batch_1" }),
|
||||
]);
|
||||
|
||||
const result = await reg.waitForMatch(undefined, 1_000);
|
||||
expect(result.reason).toBe("notify");
|
||||
// r1 was evicted by the cap; only the newest two replay.
|
||||
expect(hydrateSpy).toHaveBeenCalledWith("env_1", ["r2", "r3"], []);
|
||||
expect(evictions).toEqual(["cap"]);
|
||||
reg.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("EnvChangeRouter read-your-writes gate", () => {
|
||||
function gate(overrides: Record<string, unknown> = {}) {
|
||||
const observed: number[] = [];
|
||||
const outcomes: { outcome: string; runCount: number }[] = [];
|
||||
return {
|
||||
observed,
|
||||
outcomes,
|
||||
replicaLag: {
|
||||
getLagMs: () => 0,
|
||||
noteObservedLagMs: (ms: number) => observed.push(ms),
|
||||
marginMs: 0,
|
||||
maxDelayMs: 1_000,
|
||||
staleRetries: 3,
|
||||
onStaleHydrate: (outcome: string, runCount: number) => outcomes.push({ outcome, runCount }),
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("delays the wake hydrate by lag+margin anchored to the record's updatedAtMs", async () => {
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"] })]]);
|
||||
const g = gate({ getLagMs: () => 80, marginMs: 10 });
|
||||
const { router, src, hydrateSpy } = makeRouter(rows, { replicaLag: g.replicaLag });
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
const pushedAt = Date.now();
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: pushedAt })]);
|
||||
|
||||
// The hydrate must wait out the lag window (~90ms), not run immediately.
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
expect(hydrateSpy).not.toHaveBeenCalled();
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(Date.now() - pushedAt).toBeGreaterThanOrEqual(80);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("does not delay when the record's anchor is already past the lag window", async () => {
|
||||
// The row's updatedAt matches the watermark so the tripwire stays quiet.
|
||||
const anchorMs = Date.now() - 5_000;
|
||||
const rows = new Map([["r1", row("r1", { tags: ["a"], updatedAtMs: anchorMs })]]);
|
||||
const g = gate({ getLagMs: () => 80, marginMs: 10 });
|
||||
const { router, src } = makeRouter(rows, { replicaLag: g.replicaLag });
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
const pushedAt = Date.now();
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: anchorMs })]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(Date.now() - pushedAt).toBeLessThan(60);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("withholds a stale row, re-hydrates, and delivers only the fresh version", async () => {
|
||||
const watermark = FLOOR_MS + 10_000;
|
||||
const staleRow = row("r1", { tags: ["a"], updatedAtMs: FLOOR_MS + 5_000 });
|
||||
const freshRow = row("r1", { tags: ["a"], updatedAtMs: watermark });
|
||||
const hydrateSpy = vi
|
||||
.fn<RowHydrator["hydrateByIds"]>()
|
||||
.mockResolvedValueOnce([staleRow])
|
||||
.mockResolvedValue([freshRow]);
|
||||
const src = fakeSource();
|
||||
const g = gate();
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replicaLag: g.replicaLag,
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: watermark })]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows[0].row.updatedAt.getTime()).toBe(watermark);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(2);
|
||||
expect(g.observed.length).toBe(1); // the stale read fed the estimator
|
||||
expect(g.outcomes).toEqual([{ outcome: "recovered", runCount: 1 }]);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("a missing row with a watermark (insert race) retries and delivers when it appears", async () => {
|
||||
const watermark = FLOOR_MS + 10_000;
|
||||
const freshRow = row("r1", { tags: ["a"], updatedAtMs: watermark });
|
||||
const hydrateSpy = vi
|
||||
.fn<RowHydrator["hydrateByIds"]>()
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValue([freshRow]);
|
||||
const src = fakeSource();
|
||||
const g = gate();
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replicaLag: g.replicaLag,
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: watermark })]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.rows[0].row.id).toBe("r1");
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(2);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("delivers the stale row after exhausting retries (liveness over freshness)", async () => {
|
||||
const watermark = FLOOR_MS + 10_000;
|
||||
const staleRow = row("r1", { tags: ["a"], updatedAtMs: FLOOR_MS + 5_000 });
|
||||
const hydrateSpy = vi.fn<RowHydrator["hydrateByIds"]>().mockResolvedValue([staleRow]);
|
||||
const src = fakeSource();
|
||||
const g = gate({ staleRetries: 1 });
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replicaLag: g.replicaLag,
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: watermark })]);
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(result.rows[0].row.updatedAt.getTime()).toBe(FLOOR_MS + 5_000);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(2); // first pass + 1 retry
|
||||
expect(g.outcomes).toEqual([{ outcome: "gave_up", runCount: 1 }]);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("after giving up, echo passes deliver the fresh row once the replica catches up", async () => {
|
||||
// Recent watermark (inside the echo horizon); retries exhausted immediately.
|
||||
const watermark = Date.now() - 50;
|
||||
const staleRow = row("r1", { tags: ["a"], updatedAtMs: watermark - 5_000 });
|
||||
const freshRow = row("r1", { tags: ["a"], updatedAtMs: watermark });
|
||||
const hydrateSpy = vi
|
||||
.fn<RowHydrator["hydrateByIds"]>()
|
||||
.mockResolvedValueOnce([staleRow]) // first pass: stale -> gave_up, delivered anyway
|
||||
.mockResolvedValue([freshRow]); // echo pass: fresh
|
||||
const src = fakeSource();
|
||||
const g = gate({ staleRetries: 0 });
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replicaLag: g.replicaLag,
|
||||
});
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
|
||||
const first = reg.waitForMatch(undefined, 2_000);
|
||||
src.push("env_1", [record("r1", { tags: ["a"], updatedAtMs: watermark })]);
|
||||
const staleDelivery = await first;
|
||||
expect(staleDelivery.rows[0].row.updatedAt.getTime()).toBe(watermark - 5_000);
|
||||
expect(g.outcomes).toEqual([{ outcome: "gave_up", runCount: 1 }]);
|
||||
|
||||
// The client re-arms (as it would after consuming the stale emission); the echo
|
||||
// re-hydrate delivers the fresh version through the normal pipeline.
|
||||
const echoed = await reg.waitForMatch(undefined, 2_000);
|
||||
expect(echoed.reason).toBe("notify");
|
||||
expect(echoed.rows[0].row.updatedAt.getTime()).toBe(watermark);
|
||||
reg.close();
|
||||
});
|
||||
|
||||
it("records without a watermark bypass the tripwire entirely", async () => {
|
||||
const staleLooking = row("r1", { tags: ["a"], updatedAtMs: FLOOR_MS + 5_000 });
|
||||
const rows = new Map([["r1", staleLooking]]);
|
||||
const g = gate();
|
||||
const { router, src, hydrateSpy } = makeRouter(rows, { replicaLag: g.replicaLag });
|
||||
const reg = router.register("env_1", { kind: "tag", tags: ["a"] }, []);
|
||||
const wait = reg.waitForMatch(undefined, 2_000);
|
||||
|
||||
src.push("env_1", [record("r1", { tags: ["a"] })]); // no updatedAtMs
|
||||
|
||||
const result = await wait;
|
||||
expect(result.reason).toBe("notify");
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
expect(g.outcomes).toEqual([]);
|
||||
expect(g.observed).toEqual([]);
|
||||
reg.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,278 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { CURRENT_API_VERSION } from "~/api/versions";
|
||||
import {
|
||||
NativeRealtimeClient,
|
||||
type RealtimeListEnvironment,
|
||||
} from "~/services/realtime/nativeRealtimeClient.server";
|
||||
import { type RealtimeRunRow } from "~/services/realtime/electricStreamProtocol.server";
|
||||
import { EnvChangeRouter, type EnvChangeSource } from "~/services/realtime/envChangeRouter.server";
|
||||
import { type ChangeRecord } from "~/services/realtime/runChangeNotifier.server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const ENV: RealtimeListEnvironment = { id: "env_1", organizationId: "org_1", projectId: "proj_1" };
|
||||
|
||||
// Fixed offset floor: a row's updatedAt above/below it produces a delta / empty diff. The
|
||||
// createdAt window resolves to this same floor (large maximumCreatedAtFilterAgeMs below).
|
||||
const FLOOR_MS = Date.UTC(2026, 5, 7, 12, 0, 0);
|
||||
|
||||
function row(
|
||||
id: string,
|
||||
updatedAtMs: number,
|
||||
opts: { createdAtMs?: number; tags?: string[] } = {}
|
||||
): RealtimeRunRow {
|
||||
return {
|
||||
id,
|
||||
runTags: opts.tags ?? ["t"],
|
||||
createdAt: new Date(opts.createdAtMs ?? FLOOR_MS + 1_000),
|
||||
updatedAt: new Date(updatedAtMs),
|
||||
} as unknown as RealtimeRunRow;
|
||||
}
|
||||
|
||||
function rec(runId: string, extra: Partial<ChangeRecord> = {}): ChangeRecord {
|
||||
return { v: 1, runId, envId: "env_1", ...extra };
|
||||
}
|
||||
|
||||
/** A controllable EnvChangeSource the test pushes batches into. */
|
||||
function fakeSource() {
|
||||
const listeners = new Map<string, Set<(records: ChangeRecord[]) => void>>();
|
||||
const source: EnvChangeSource = {
|
||||
subscribeToEnv(envId, onBatch) {
|
||||
let set = listeners.get(envId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(envId, set);
|
||||
}
|
||||
set.add(onBatch);
|
||||
return () => listeners.get(envId)?.delete(onBatch);
|
||||
},
|
||||
};
|
||||
return {
|
||||
source,
|
||||
push: (envId: string, records: ChangeRecord[]) => {
|
||||
for (const l of listeners.get(envId) ?? []) l(records);
|
||||
},
|
||||
isSubscribed: (envId: string) => (listeners.get(envId)?.size ?? 0) > 0,
|
||||
};
|
||||
}
|
||||
|
||||
function makeClient(overrides: Record<string, unknown> = {}) {
|
||||
let rowsToReturn: RealtimeRunRow[] = [];
|
||||
const hydrateSpy = vi.fn(async (_env: string, ids: string[]) =>
|
||||
rowsToReturn.filter((r) => ids.includes(r.id))
|
||||
);
|
||||
const resolveSpy = vi.fn(async () => rowsToReturn.map((r) => r.id));
|
||||
const src = fakeSource();
|
||||
const router = new EnvChangeRouter({
|
||||
source: src.source,
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replayWindowMs: 0,
|
||||
unsubscribeLingerMs: 0,
|
||||
...((overrides.routerOptions as Record<string, unknown>) ?? {}),
|
||||
});
|
||||
delete overrides.routerOptions;
|
||||
|
||||
const client = new NativeRealtimeClient({
|
||||
runReader: { getRunById: async () => null, hydrateByIds: hydrateSpy } as any,
|
||||
runListResolver: { resolveMatchingRunIds: resolveSpy } as any,
|
||||
router,
|
||||
limiter: { incrementAndCheck: async () => true, decrement: async () => {} } as any,
|
||||
cachedLimitProvider: { getCachedLimit: async () => 100 },
|
||||
// Large so the recovered createdAt floor isn't clamped past FLOOR_MS.
|
||||
maximumCreatedAtFilterAgeMs: 100 * 365 * 24 * 60 * 60 * 1000,
|
||||
runSetResolveCacheTtlMs: 0,
|
||||
livePollTimeoutMs: 10_000,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
return {
|
||||
client,
|
||||
src,
|
||||
hydrateSpy,
|
||||
resolveSpy,
|
||||
setRows: (rows: RealtimeRunRow[]) => (rowsToReturn = rows),
|
||||
};
|
||||
}
|
||||
|
||||
function liveRuns(client: NativeRealtimeClient) {
|
||||
return client.streamRuns(
|
||||
`http://localhost:3030/realtime/v1/runs?offset=${FLOOR_MS}_1&live=true&handle=runs_${FLOOR_MS}_7`,
|
||||
ENV,
|
||||
{ tags: ["t"] },
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
}
|
||||
|
||||
async function whenWaiting(src: ReturnType<typeof fakeSource>) {
|
||||
// Subscribed (feed registered) + a tick so waitForMatch has armed feed.resolve.
|
||||
await vi.waitFor(() => expect(src.isSubscribed("env_1")).toBe(true));
|
||||
await sleep(15);
|
||||
}
|
||||
|
||||
async function bodyOf(res: Response) {
|
||||
return JSON.parse(await res.text()) as Array<{
|
||||
headers?: { control?: string; operation?: string };
|
||||
value?: unknown;
|
||||
}>;
|
||||
}
|
||||
const hasRowOp = (body: Awaited<ReturnType<typeof bodyOf>>) =>
|
||||
body.some((m) => m?.headers?.operation || (m && typeof m === "object" && "value" in m));
|
||||
const isUpToDate = (body: Awaited<ReturnType<typeof bodyOf>>) =>
|
||||
body.some((m) => m?.headers?.control === "up-to-date");
|
||||
|
||||
describe("NativeRealtimeClient multi-run live path over the router", () => {
|
||||
it("a matching change hydrates by id (no ClickHouse) and returns a delta", async () => {
|
||||
const emits: Array<[string, number, number]> = [];
|
||||
const { client, src, hydrateSpy, resolveSpy, setRows } = makeClient({
|
||||
onEmit: (path: string, lagMs: number, rows: number) => emits.push([path, lagMs, rows]),
|
||||
});
|
||||
setRows([row("run_1", FLOOR_MS + 5_000, { tags: ["t"] })]);
|
||||
|
||||
const responsePromise = liveRuns(client);
|
||||
await whenWaiting(src);
|
||||
src.push("env_1", [rec("run_1", { tags: ["t", "x"] })]);
|
||||
|
||||
const res = await responsePromise;
|
||||
expect(res.status).toBe(200);
|
||||
expect(hasRowOp(await bodyOf(res))).toBe(true);
|
||||
expect(resolveSpy).not.toHaveBeenCalled(); // ClickHouse skipped
|
||||
expect(hydrateSpy).toHaveBeenCalledWith("env_1", ["run_1"], expect.anything());
|
||||
expect(emits).toHaveLength(1);
|
||||
expect(emits[0][0]).toBe("fast-hydrate");
|
||||
expect(emits[0][2]).toBe(1); // one delta row
|
||||
});
|
||||
|
||||
it("a change that doesn't match the filter never wakes the feed (no CH, no PG); a later match does", async () => {
|
||||
const { client, src, hydrateSpy, resolveSpy, setRows } = makeClient();
|
||||
setRows([row("run_1", FLOOR_MS + 5_000, { tags: ["t"] })]);
|
||||
|
||||
const responsePromise = liveRuns(client);
|
||||
let settled = false;
|
||||
void responsePromise.then(() => (settled = true));
|
||||
await whenWaiting(src);
|
||||
|
||||
src.push("env_1", [rec("run_x", { tags: ["other"] })]); // doesn't intersect ["t"]
|
||||
await sleep(50);
|
||||
expect(settled).toBe(false);
|
||||
expect(hydrateSpy).not.toHaveBeenCalled(); // router never routed it
|
||||
expect(resolveSpy).not.toHaveBeenCalled();
|
||||
|
||||
src.push("env_1", [rec("run_1", { tags: ["t"] })]);
|
||||
const res = await responsePromise;
|
||||
expect(settled).toBe(true);
|
||||
expect(hasRowOp(await bodyOf(res))).toBe(true);
|
||||
});
|
||||
|
||||
it("a matching run created before the window floor is hydrated but dropped (keeps holding)", async () => {
|
||||
// Generous backstop so the "still holding" assertion can't race a timeout in slow CI.
|
||||
const { client, src, hydrateSpy, resolveSpy, setRows } = makeClient({
|
||||
livePollTimeoutMs: 1500,
|
||||
});
|
||||
setRows([row("run_1", FLOOR_MS + 5_000, { createdAtMs: FLOOR_MS - 10_000, tags: ["t"] })]);
|
||||
|
||||
const responsePromise = liveRuns(client);
|
||||
let settled = false;
|
||||
void responsePromise.then(() => (settled = true));
|
||||
await whenWaiting(src);
|
||||
src.push("env_1", [rec("run_1", { tags: ["t"] })]);
|
||||
|
||||
await sleep(40);
|
||||
expect(settled).toBe(false); // dropped by the createdAt floor -> held
|
||||
expect(hydrateSpy).toHaveBeenCalledWith("env_1", ["run_1"], expect.anything());
|
||||
expect(resolveSpy).not.toHaveBeenCalled();
|
||||
|
||||
await responsePromise; // drain via the backstop
|
||||
});
|
||||
|
||||
it("the backstop timeout does a full ClickHouse resolve and returns up-to-date", async () => {
|
||||
const backstopResults: string[] = [];
|
||||
const { client, resolveSpy } = makeClient({
|
||||
livePollTimeoutMs: 50,
|
||||
onBackstopResult: (r: string) => backstopResults.push(r),
|
||||
});
|
||||
const res = await liveRuns(client); // never pushed -> backstop fires
|
||||
expect(res.status).toBe(200);
|
||||
expect(isUpToDate(await bodyOf(res))).toBe(true);
|
||||
expect(resolveSpy).toHaveBeenCalled();
|
||||
expect(backstopResults).toEqual(["empty"]);
|
||||
});
|
||||
|
||||
it("a cold env registration resolves immediately instead of holding blind", async () => {
|
||||
// Fresh env subscription (gapCovered=false): a change in the inter-poll gap may have
|
||||
// been missed, so the live poll probes once. The row advanced past the offset floor.
|
||||
const { client, resolveSpy, setRows } = makeClient({
|
||||
routerOptions: { replayWindowMs: 2_000 },
|
||||
});
|
||||
setRows([row("run_1", FLOOR_MS + 5_000, { tags: ["t"] })]);
|
||||
|
||||
const res = await liveRuns(client); // no push needed — the cold probe finds the delta
|
||||
expect(res.status).toBe(200);
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(1);
|
||||
expect(hasRowOp(await bodyOf(res))).toBe(true);
|
||||
});
|
||||
|
||||
it("a cold probe with nothing missed keeps holding", async () => {
|
||||
const { client, src, resolveSpy, setRows } = makeClient({
|
||||
routerOptions: { replayWindowMs: 2_000 },
|
||||
livePollTimeoutMs: 1_500,
|
||||
});
|
||||
setRows([row("run_1", FLOOR_MS - 1_000, { tags: ["t"] })]); // at/below the offset floor
|
||||
|
||||
const responsePromise = liveRuns(client);
|
||||
let settled = false;
|
||||
void responsePromise.then(() => (settled = true));
|
||||
await whenWaiting(src);
|
||||
await sleep(50);
|
||||
expect(settled).toBe(false); // probed, found nothing missed, held
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(1);
|
||||
await responsePromise; // drain via the backstop
|
||||
});
|
||||
|
||||
it("a single-run poll holds on a replayed already-seen record instead of busy re-polling", async () => {
|
||||
const { client, src, setRows } = makeClient({
|
||||
routerOptions: { replayWindowMs: 2_000 },
|
||||
livePollTimeoutMs: 300,
|
||||
});
|
||||
setRows([row("run_1", FLOOR_MS + 1_000)]);
|
||||
const url = `http://localhost:3030/realtime/v1/runs/run_1?offset=${FLOOR_MS + 1_000}_1&handle=run-run_1&live=true`;
|
||||
|
||||
// First poll subscribes the env, then drains via its backstop.
|
||||
const first = await client.streamRun(
|
||||
url,
|
||||
ENV,
|
||||
"run_1",
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
// The record lands between polls; the lingering env subscription buffers it.
|
||||
src.push("env_1", [rec("run_1")]);
|
||||
|
||||
// The next poll replays it, but the row hasn't advanced past the client's offset:
|
||||
// the poll must HOLD (the old behavior returned up-to-date instantly = a busy loop).
|
||||
let settled = false;
|
||||
const second = client.streamRun(url, ENV, "run_1", CURRENT_API_VERSION, undefined, "1.0.0");
|
||||
void second.then(() => (settled = true));
|
||||
await sleep(120);
|
||||
expect(settled).toBe(false);
|
||||
expect((await second).status).toBe(200); // drains via the backstop
|
||||
});
|
||||
|
||||
it("with holdOnEmpty=false, a matched-but-not-advanced change returns up-to-date without ClickHouse", async () => {
|
||||
const { client, src, resolveSpy, setRows } = makeClient({ holdOnEmpty: false });
|
||||
// Matches the tag and is in-window, but updatedAt is at/below the offset floor -> no delta.
|
||||
setRows([row("run_1", FLOOR_MS - 1_000, { tags: ["t"] })]);
|
||||
|
||||
const responsePromise = liveRuns(client);
|
||||
await whenWaiting(src);
|
||||
src.push("env_1", [rec("run_1", { tags: ["t"] })]);
|
||||
|
||||
const res = await responsePromise;
|
||||
expect(res.status).toBe(200);
|
||||
expect(isUpToDate(await bodyOf(res))).toBe(true);
|
||||
expect(resolveSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { CURRENT_API_VERSION } from "~/api/versions";
|
||||
import {
|
||||
NativeRealtimeClient,
|
||||
type RealtimeListEnvironment,
|
||||
} from "~/services/realtime/nativeRealtimeClient.server";
|
||||
import { type RealtimeRunRow } from "~/services/realtime/electricStreamProtocol.server";
|
||||
import { EnvChangeRouter } from "~/services/realtime/envChangeRouter.server";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
function sampleRow(): RealtimeRunRow {
|
||||
return {
|
||||
id: "run_1",
|
||||
taskIdentifier: "t",
|
||||
createdAt: new Date("2026-06-07T10:00:00.000Z"),
|
||||
updatedAt: new Date("2026-06-07T10:00:01.000Z"),
|
||||
startedAt: null,
|
||||
delayUntil: null,
|
||||
queuedAt: null,
|
||||
expiredAt: null,
|
||||
completedAt: null,
|
||||
friendlyId: "run_friendly_1",
|
||||
number: 1,
|
||||
isTest: false,
|
||||
status: "EXECUTING",
|
||||
usageDurationMs: 0,
|
||||
costInCents: 0,
|
||||
baseCostInCents: 0,
|
||||
ttl: null,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
metadata: null,
|
||||
metadataType: "application/json",
|
||||
output: null,
|
||||
outputType: "application/json",
|
||||
runTags: [],
|
||||
error: null,
|
||||
realtimeStreams: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Only the initial-snapshot path is exercised here, which touches the shared
|
||||
// #buildResponse — enough to lock the response-header contract.
|
||||
function makeClient(row: RealtimeRunRow | null) {
|
||||
return new NativeRealtimeClient({
|
||||
runReader: {
|
||||
getRunById: async () => row,
|
||||
hydrateByIds: async () => (row ? [row] : []),
|
||||
} as any,
|
||||
runListResolver: { resolveMatchingRunIds: async () => [] } as any,
|
||||
// Snapshot path only; the router (over a no-op source) is never invoked here.
|
||||
router: new EnvChangeRouter({
|
||||
source: { subscribeToEnv: () => () => {} },
|
||||
hydrator: { hydrateByIds: async () => (row ? [row] : []) },
|
||||
replayWindowMs: 0,
|
||||
unsubscribeLingerMs: 0,
|
||||
}),
|
||||
limiter: { incrementAndCheck: async () => true, decrement: async () => {} } as any,
|
||||
cachedLimitProvider: { getCachedLimit: async () => 100 },
|
||||
maximumCreatedAtFilterAgeMs: 24 * 60 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
const ENV: RealtimeListEnvironment = {
|
||||
id: "env_1",
|
||||
organizationId: "org_1",
|
||||
projectId: "proj_1",
|
||||
};
|
||||
|
||||
describe("NativeRealtimeClient response headers", () => {
|
||||
it("exposes electric headers cross-origin so browser hooks can read them", async () => {
|
||||
const client = makeClient(sampleRow());
|
||||
const res = await client.streamRun(
|
||||
"http://localhost:3030/realtime/v1/runs/run_1?offset=-1",
|
||||
ENV,
|
||||
"run_1",
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0-beta.1" // modern client => lowercase electric-* headers
|
||||
);
|
||||
|
||||
// Without these the deployed @electric-sql/client throws MissingHeadersError
|
||||
// (it can't read the electric-* headers across origins). This regressed once.
|
||||
expect(res.headers.get("access-control-allow-origin")).toBe("*");
|
||||
expect(res.headers.get("access-control-expose-headers")).toBe("*");
|
||||
|
||||
// Initial (non-live) snapshot requires offset + handle + schema.
|
||||
expect(res.headers.get("electric-offset")).toBeTruthy();
|
||||
expect(res.headers.get("electric-handle")).toBeTruthy();
|
||||
expect(res.headers.get("electric-schema")).toBeTruthy();
|
||||
expect(res.headers.get("content-type")).toBe("application/json");
|
||||
});
|
||||
|
||||
it("renames headers for legacy (0.4.0) clients", async () => {
|
||||
const client = makeClient(sampleRow());
|
||||
const res = await client.streamRun(
|
||||
"http://localhost:3030/realtime/v1/runs/run_1?offset=-1",
|
||||
ENV,
|
||||
"run_1",
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
undefined // no client version => legacy header names
|
||||
);
|
||||
|
||||
expect(res.headers.get("electric-chunk-last-offset")).toBeTruthy();
|
||||
expect(res.headers.get("electric-shape-id")).toBeTruthy();
|
||||
expect(res.headers.get("electric-offset")).toBeNull();
|
||||
expect(res.headers.get("electric-handle")).toBeNull();
|
||||
expect(res.headers.get("access-control-expose-headers")).toBe("*");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,347 @@
|
||||
import { CURRENT_API_VERSION } from "~/api/versions";
|
||||
import {
|
||||
NativeRealtimeClient,
|
||||
type RealtimeListEnvironment,
|
||||
} from "~/services/realtime/nativeRealtimeClient.server";
|
||||
import { type RealtimeRunRow } from "~/services/realtime/electricStreamProtocol.server";
|
||||
import { EnvChangeRouter } from "~/services/realtime/envChangeRouter.server";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const ENV: RealtimeListEnvironment = { id: "env_1", organizationId: "org_1", projectId: "proj_1" };
|
||||
|
||||
function row(id: string): RealtimeRunRow {
|
||||
// Only id/createdAt/updatedAt are read directly; the rest serialize to null.
|
||||
return {
|
||||
id,
|
||||
createdAt: new Date("2026-06-07T09:00:00.000Z"),
|
||||
updatedAt: new Date("2026-06-07T10:00:00.000Z"),
|
||||
} as unknown as RealtimeRunRow;
|
||||
}
|
||||
|
||||
function makeClient(overrides: Record<string, unknown> = {}) {
|
||||
const resolveSpy = vi.fn(async () => ["run_1", "run_2"]);
|
||||
const hydrateSpy = vi.fn(async (_env: string, ids: string[]) => ids.map(row));
|
||||
|
||||
const client = new NativeRealtimeClient({
|
||||
runReader: { getRunById: async () => null, hydrateByIds: hydrateSpy } as any,
|
||||
runListResolver: { resolveMatchingRunIds: resolveSpy } as any,
|
||||
// No-op source: live polls never get a router wake, so they fall through to the
|
||||
// backstop full-resolve — which is what the live tests below assert on.
|
||||
router: new EnvChangeRouter({
|
||||
source: { subscribeToEnv: () => () => {} },
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replayWindowMs: 0,
|
||||
unsubscribeLingerMs: 0,
|
||||
}),
|
||||
limiter: { incrementAndCheck: async () => true, decrement: async () => {} } as any,
|
||||
cachedLimitProvider: { getCachedLimit: async () => 100 },
|
||||
maximumCreatedAtFilterAgeMs: 24 * 60 * 60 * 1000,
|
||||
runSetResolveCacheTtlMs: 5_000,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
return { client, resolveSpy, hydrateSpy };
|
||||
}
|
||||
|
||||
// streamBatch with offset=-1 takes the snapshot path, which calls the coalescing
|
||||
// resolve+hydrate directly (no concurrency slot / subscription needed).
|
||||
function snapshot(client: NativeRealtimeClient, batchId: string, skipColumns?: string) {
|
||||
const skip = skipColumns ? `&skipColumns=${skipColumns}` : "";
|
||||
return client.streamBatch(
|
||||
`http://localhost:3030/realtime/v1/batches/${batchId}?offset=-1${skip}`,
|
||||
ENV,
|
||||
batchId,
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
}
|
||||
|
||||
// Tag-list snapshot (offset=-1) — exercises the createdAt bucketing + cache key.
|
||||
function snapshotTag(client: NativeRealtimeClient, tags: string[]) {
|
||||
return client.streamRuns(
|
||||
"http://localhost:3030/realtime/v1/runs?offset=-1",
|
||||
ENV,
|
||||
{ tags },
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
}
|
||||
|
||||
describe("NativeRealtimeClient run-set resolve coalescing + cache", () => {
|
||||
it("coalesces concurrent same-filter resolves into one ClickHouse + Postgres query", async () => {
|
||||
const { client, resolveSpy, hydrateSpy } = makeClient();
|
||||
let release!: (ids: string[]) => void;
|
||||
const gate = new Promise<string[]>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
resolveSpy.mockReturnValueOnce(gate);
|
||||
|
||||
const p1 = snapshot(client, "batch_1");
|
||||
const p2 = snapshot(client, "batch_1");
|
||||
release(["run_1"]);
|
||||
await Promise.all([p1, p2]);
|
||||
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(1);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("serves a second same-filter request from the cache within the TTL", async () => {
|
||||
const { client, resolveSpy, hydrateSpy } = makeClient();
|
||||
await snapshot(client, "batch_1");
|
||||
await snapshot(client, "batch_1");
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(1);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not share the cache across different filters", async () => {
|
||||
const { client, resolveSpy } = makeClient();
|
||||
await snapshot(client, "batch_1");
|
||||
await snapshot(client, "batch_2");
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("re-queries after the cache TTL expires", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
try {
|
||||
const { client, resolveSpy } = makeClient({ runSetResolveCacheTtlMs: 1_000 });
|
||||
await snapshot(client, "batch_1");
|
||||
vi.advanceTimersByTime(1_001);
|
||||
await snapshot(client, "batch_1");
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes the client's skipColumns through to the hydrator (column projection)", async () => {
|
||||
const { client, hydrateSpy } = makeClient();
|
||||
await snapshot(client, "batch_1", "payload,output");
|
||||
expect(hydrateSpy).toHaveBeenCalledWith("env_1", expect.any(Array), ["payload", "output"]);
|
||||
});
|
||||
|
||||
it("reports resolve outcomes (miss then hit) to the metrics hook", async () => {
|
||||
const results: string[] = [];
|
||||
const { client } = makeClient({ onRunSetResolve: (r: string) => results.push(r) });
|
||||
await snapshot(client, "batch_1");
|
||||
await snapshot(client, "batch_1");
|
||||
expect(results).toEqual(["miss", "hit"]);
|
||||
});
|
||||
|
||||
it("mints a distinct batch handle per connection and echoes a client-provided one", async () => {
|
||||
const { client } = makeClient();
|
||||
// Two subscribers to the SAME batch must never share a handle (the working-set
|
||||
// cache is keyed by it; sharing lets one suppress the other's deltas forever).
|
||||
const res1 = await snapshot(client, "batch_1");
|
||||
const res2 = await snapshot(client, "batch_1");
|
||||
const h1 = res1.headers.get("electric-handle");
|
||||
const h2 = res2.headers.get("electric-handle");
|
||||
expect(h1).toBeTruthy();
|
||||
expect(h1).not.toBe(h2);
|
||||
|
||||
// Catch-up under an existing handle keeps it.
|
||||
const res3 = await client.streamBatch(
|
||||
`http://localhost:3030/realtime/v1/batches/batch_1?offset=123_1&handle=${h1}`,
|
||||
ENV,
|
||||
"batch_1",
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
expect(res3.headers.get("electric-handle")).toBe(h1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("NativeRealtimeClient resolve admission gate (mass-reconnect stampede)", () => {
|
||||
// A resolver that blocks each invocation until released, so we can watch how many run
|
||||
// concurrently. Tracks peak concurrency and exposes a release-one-at-a-time drain.
|
||||
function gatedResolver() {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
const resolve = vi.fn(async () => {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise<void>((r) => releases.push(r));
|
||||
active--;
|
||||
return ["run_1"];
|
||||
});
|
||||
return {
|
||||
resolve,
|
||||
peak: () => peak,
|
||||
releaseOne: () => releases.shift()?.(),
|
||||
waiting: () => releases.length,
|
||||
};
|
||||
}
|
||||
|
||||
function makeGatedClient(
|
||||
resolveAdmissionLimit: number,
|
||||
resolver: ReturnType<typeof gatedResolver>
|
||||
) {
|
||||
const hydrateSpy = vi.fn(async (_env: string, ids: string[]) => ids.map(row));
|
||||
return new NativeRealtimeClient({
|
||||
runReader: { getRunById: async () => null, hydrateByIds: hydrateSpy } as any,
|
||||
runListResolver: { resolveMatchingRunIds: resolver.resolve } as any,
|
||||
router: new EnvChangeRouter({
|
||||
source: { subscribeToEnv: () => () => {} },
|
||||
hydrator: { hydrateByIds: hydrateSpy },
|
||||
replayWindowMs: 0,
|
||||
unsubscribeLingerMs: 0,
|
||||
}),
|
||||
limiter: { incrementAndCheck: async () => true, decrement: async () => {} } as any,
|
||||
cachedLimitProvider: { getCachedLimit: async () => 100 },
|
||||
maximumCreatedAtFilterAgeMs: 24 * 60 * 60 * 1000,
|
||||
runSetResolveCacheTtlMs: 0, // no cache -> every distinct filter is a fresh resolve
|
||||
resolveAdmissionLimit,
|
||||
});
|
||||
}
|
||||
|
||||
it("throttles a distinct-filter stampede to the admission limit of concurrent CH resolves", async () => {
|
||||
const resolver = gatedResolver();
|
||||
const client = makeGatedClient(2, resolver);
|
||||
|
||||
// 5 distinct batchIds => 5 distinct filters => 5 fresh resolves, fired at once.
|
||||
const polls = [0, 1, 2, 3, 4].map((i) => snapshot(client, `batch_${i}`));
|
||||
|
||||
// Only the limit (2) may run concurrently; the rest queue for a permit.
|
||||
await vi.waitFor(() => expect(resolver.resolve).toHaveBeenCalledTimes(2));
|
||||
await sleep(20);
|
||||
expect(resolver.resolve).toHaveBeenCalledTimes(2); // 3 still queued behind the gate
|
||||
expect(resolver.peak()).toBe(2);
|
||||
|
||||
// Drain: each release frees a permit, admitting exactly one queued resolve.
|
||||
while (resolver.waiting() > 0) {
|
||||
resolver.releaseOne();
|
||||
await sleep(5);
|
||||
}
|
||||
await Promise.all(polls);
|
||||
|
||||
expect(resolver.resolve).toHaveBeenCalledTimes(5); // all ran...
|
||||
expect(resolver.peak()).toBe(2); // ...but never more than the limit at once
|
||||
});
|
||||
|
||||
it("lets a same-filter burst through on a single permit (coalesces before the gate)", async () => {
|
||||
const resolver = gatedResolver();
|
||||
const client = makeGatedClient(1, resolver); // limit 1 would deadlock if each took a permit
|
||||
|
||||
// 5 identical filters fired at once -> single-flight collapses to one in-flight resolve.
|
||||
const polls = [0, 1, 2, 3, 4].map(() => snapshot(client, "batch_same"));
|
||||
await vi.waitFor(() => expect(resolver.resolve).toHaveBeenCalledTimes(1));
|
||||
await sleep(20);
|
||||
|
||||
resolver.releaseOne();
|
||||
await Promise.all(polls);
|
||||
expect(resolver.resolve).toHaveBeenCalledTimes(1); // one resolve, one permit, no queue
|
||||
});
|
||||
});
|
||||
|
||||
describe("NativeRealtimeClient tag-list createdAt bucketing", () => {
|
||||
it("floors the resolved createdAt lower bound to the bucket boundary", async () => {
|
||||
// Fix the clock to a non-bucket-aligned instant so the assertion is deterministic.
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date("2026-06-07T10:00:30.500Z"));
|
||||
try {
|
||||
const { client, resolveSpy } = makeClient({ runSetCreatedAtBucketMs: 60_000 });
|
||||
await snapshotTag(client, ["critical"]);
|
||||
const passed = resolveSpy.mock.calls[0][0].createdAtAfter as Date;
|
||||
expect(passed.getTime() % 60_000).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("lets two same-tag feeds in the same bucket share one resolve", async () => {
|
||||
// A large bucket guarantees both windows floor to the same boundary regardless of
|
||||
// the sub-millisecond gap between the two calls.
|
||||
const { client, resolveSpy, hydrateSpy } = makeClient({
|
||||
runSetCreatedAtBucketMs: 60 * 60_000,
|
||||
});
|
||||
await snapshotTag(client, ["critical"]);
|
||||
await snapshotTag(client, ["critical"]);
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(1);
|
||||
expect(hydrateSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not share across different tags", async () => {
|
||||
const { client, resolveSpy } = makeClient({ runSetCreatedAtBucketMs: 60 * 60_000 });
|
||||
await snapshotTag(client, ["critical"]);
|
||||
await snapshotTag(client, ["debug"]);
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not collide a comma-containing tag with two separate tags", async () => {
|
||||
const { client, resolveSpy } = makeClient({ runSetCreatedAtBucketMs: 60 * 60_000 });
|
||||
await snapshotTag(client, ["a,b"]); // one tag "a,b"
|
||||
await snapshotTag(client, ["a", "b"]); // two tags a OR b — a different filter
|
||||
expect(resolveSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps each feed's exact lower bound when bucketing is disabled (0)", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date("2026-06-07T10:00:30.500Z"));
|
||||
try {
|
||||
const { client, resolveSpy } = makeClient({ runSetCreatedAtBucketMs: 0 });
|
||||
await snapshotTag(client, ["critical"]);
|
||||
const passed = resolveSpy.mock.calls[0][0].createdAtAfter as Date;
|
||||
// Exact (now - 24h) lower bound, not floored to a 60s boundary.
|
||||
expect(passed.getTime() % 60_000).not.toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("NativeRealtimeClient review fixes", () => {
|
||||
// makeClient's router has a no-op source, so the live poll never gets a wake and falls
|
||||
// through to its backstop timeout — the full ClickHouse resolve these tests assert on
|
||||
// (createdAt clamp / concurrency limit).
|
||||
|
||||
it("clamps a stale/crafted handle's createdAt up to the max-age floor", async () => {
|
||||
const maxAge = 24 * 60 * 60 * 1000;
|
||||
const { client, resolveSpy } = makeClient({
|
||||
maximumCreatedAtFilterAgeMs: maxAge,
|
||||
runSetCreatedAtBucketMs: 0,
|
||||
livePollTimeoutMs: 50,
|
||||
});
|
||||
const before = Date.now();
|
||||
// Handle encodes createdAt = 1ms epoch, far older than the 24h ceiling.
|
||||
await client.streamRuns(
|
||||
"http://localhost:3030/realtime/v1/runs?offset=123_1&live=true&handle=runs_1_7",
|
||||
ENV,
|
||||
{ tags: ["t"] },
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
const passed = resolveSpy.mock.calls[0][0].createdAtAfter as Date;
|
||||
// Clamped to ~now - maxAge, not the epoch value encoded in the handle.
|
||||
expect(passed.getTime()).toBeGreaterThan(before - maxAge - 1_000);
|
||||
});
|
||||
|
||||
it("enforces a concurrency limit of 0 instead of failing with a 500", async () => {
|
||||
let limitCheckedWith: number | undefined;
|
||||
const { client } = makeClient({
|
||||
cachedLimitProvider: { getCachedLimit: async () => 0 },
|
||||
limiter: {
|
||||
incrementAndCheck: async (_env: string, _id: string, limit: number) => {
|
||||
limitCheckedWith = limit;
|
||||
return true;
|
||||
},
|
||||
decrement: async () => {},
|
||||
},
|
||||
livePollTimeoutMs: 50,
|
||||
});
|
||||
const res = await client.streamBatch(
|
||||
"http://localhost:3030/realtime/v1/batches/batch_1?offset=123_1&live=true&handle=batch_batch_1_7_abc",
|
||||
ENV,
|
||||
"batch_1",
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(limitCheckedWith).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
import { redisTest } from "@internal/testcontainers";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { CURRENT_API_VERSION } from "~/api/versions";
|
||||
import { EnvChangeRouter } from "~/services/realtime/envChangeRouter.server";
|
||||
import {
|
||||
NativeRealtimeClient,
|
||||
type RealtimeListEnvironment,
|
||||
} from "~/services/realtime/nativeRealtimeClient.server";
|
||||
import {
|
||||
InMemoryReplayCursorStore,
|
||||
RedisReplayCursorStore,
|
||||
type ReplayCursorStore,
|
||||
} from "~/services/realtime/replayCursorStore.server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
describe("InMemoryReplayCursorStore", () => {
|
||||
it("round-trips and expires", async () => {
|
||||
const store = new InMemoryReplayCursorStore(50, 10);
|
||||
store.set("env_1:h1", 123_456);
|
||||
expect(await store.get("env_1:h1")).toBe(123_456);
|
||||
expect(await store.get("env_1:other")).toBeUndefined();
|
||||
await sleep(60);
|
||||
expect(await store.get("env_1:h1")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("RedisReplayCursorStore", () => {
|
||||
redisTest("round-trips, misses, and expires via PX", async ({ redisOptions }) => {
|
||||
const store = new RedisReplayCursorStore({
|
||||
redis: { ...redisOptions, tlsDisabled: true },
|
||||
ttlMs: 150,
|
||||
});
|
||||
try {
|
||||
const now = Date.now();
|
||||
store.set("env_1:h1", now);
|
||||
await vi.waitFor(async () => expect(await store.get("env_1:h1")).toBe(now));
|
||||
expect(await store.get("env_1:missing")).toBeUndefined();
|
||||
await sleep(200);
|
||||
expect(await store.get("env_1:h1")).toBeUndefined();
|
||||
} finally {
|
||||
await store.quit();
|
||||
}
|
||||
});
|
||||
|
||||
redisTest(
|
||||
"a second store instance reads the first's cursor (fleet sharing)",
|
||||
async ({ redisOptions }) => {
|
||||
const a = new RedisReplayCursorStore({
|
||||
redis: { ...redisOptions, tlsDisabled: true },
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
const b = new RedisReplayCursorStore({
|
||||
redis: { ...redisOptions, tlsDisabled: true },
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
try {
|
||||
a.set("env_1:h2", 42_000);
|
||||
await vi.waitFor(async () => expect(await b.get("env_1:h2")).toBe(42_000));
|
||||
} finally {
|
||||
await Promise.all([a.quit(), b.quit()]);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it("degrades to undefined within the read deadline when Redis is unreachable", async () => {
|
||||
const results: Array<[string, boolean]> = [];
|
||||
const store = new RedisReplayCursorStore({
|
||||
redis: { host: "127.0.0.1", port: 1, tlsDisabled: true } as any,
|
||||
ttlMs: 1_000,
|
||||
getTimeoutMs: 100,
|
||||
onResult: (op, ok) => results.push([op, ok]),
|
||||
});
|
||||
try {
|
||||
expect(await store.get("env_1:h3")).toBeUndefined();
|
||||
expect(results).toContainEqual(["get", false]);
|
||||
} finally {
|
||||
await store.quit().catch(() => {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("NativeRealtimeClient replay-cursor threading", () => {
|
||||
const ENV: RealtimeListEnvironment = {
|
||||
id: "env_1",
|
||||
organizationId: "org_1",
|
||||
projectId: "proj_1",
|
||||
};
|
||||
const FLOOR_MS = Date.UTC(2026, 5, 7, 12, 0, 0);
|
||||
|
||||
it("passes the stored cursor to register and stamps the store after responding", async () => {
|
||||
const cursorMs = Date.now() - 500;
|
||||
const gets: string[] = [];
|
||||
const sets: Array<[string, number]> = [];
|
||||
const store: ReplayCursorStore = {
|
||||
get: async (key) => {
|
||||
gets.push(key);
|
||||
return cursorMs;
|
||||
},
|
||||
set: (key, ms) => {
|
||||
sets.push([key, ms]);
|
||||
},
|
||||
};
|
||||
|
||||
const router = new EnvChangeRouter({
|
||||
source: { subscribeToEnv: () => () => {} },
|
||||
hydrator: { hydrateByIds: async () => [] },
|
||||
replayWindowMs: 0,
|
||||
unsubscribeLingerMs: 0,
|
||||
});
|
||||
const registerSpy = vi.spyOn(router, "register");
|
||||
|
||||
const client = new NativeRealtimeClient({
|
||||
runReader: { getRunById: async () => null, hydrateByIds: async () => [] } as any,
|
||||
runListResolver: { resolveMatchingRunIds: async () => [] } as any,
|
||||
router,
|
||||
limiter: { incrementAndCheck: async () => true, decrement: async () => {} } as any,
|
||||
cachedLimitProvider: { getCachedLimit: async () => 100 },
|
||||
maximumCreatedAtFilterAgeMs: 100 * 365 * 24 * 60 * 60 * 1000,
|
||||
runSetResolveCacheTtlMs: 0,
|
||||
livePollTimeoutMs: 30,
|
||||
replayCursorStore: store,
|
||||
});
|
||||
|
||||
const res = await client.streamRuns(
|
||||
`http://localhost:3030/realtime/v1/runs?offset=${FLOOR_MS}_1&live=true&handle=runs_${FLOOR_MS}_7`,
|
||||
ENV,
|
||||
{ tags: ["t"] },
|
||||
CURRENT_API_VERSION,
|
||||
undefined,
|
||||
"1.0.0"
|
||||
);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(gets).toEqual([`env_1:runs_${FLOOR_MS}_7`]);
|
||||
expect(registerSpy).toHaveBeenCalledWith(
|
||||
"env_1",
|
||||
expect.objectContaining({ kind: "tag" }),
|
||||
expect.anything(),
|
||||
{ replaySinceMs: cursorMs }
|
||||
);
|
||||
// The backstop's up-to-date response stamps the cursor for the next poll.
|
||||
expect(sets.length).toBe(1);
|
||||
expect(sets[0][0]).toBe(`env_1:runs_${FLOOR_MS}_7`);
|
||||
expect(sets[0][1]).toBeGreaterThanOrEqual(cursorMs);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,164 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
FirstSupportedReplicaLagSource,
|
||||
ReplicaLagEstimator,
|
||||
type ReplicaLagSource,
|
||||
} from "~/services/realtime/replicaLagEstimator.server";
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
function source(sampleLagMs: () => Promise<number | undefined>, name = "fake"): ReplicaLagSource {
|
||||
return { name, sampleLagMs };
|
||||
}
|
||||
|
||||
describe("ReplicaLagEstimator", () => {
|
||||
let estimator: ReplicaLagEstimator | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
estimator?.stop();
|
||||
estimator = undefined;
|
||||
});
|
||||
|
||||
it("returns the default before any sample lands", () => {
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => undefined),
|
||||
defaultLagMs: 42,
|
||||
});
|
||||
expect(estimator.getLagMs()).toBe(42);
|
||||
});
|
||||
|
||||
it("samples the source while touched and reports the window max", async () => {
|
||||
const samples = [10, 60, 20];
|
||||
let i = 0;
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => samples[Math.min(i++, samples.length - 1)]),
|
||||
sampleIntervalMs: 10,
|
||||
windowMs: 5_000,
|
||||
defaultLagMs: 0,
|
||||
});
|
||||
estimator.touch();
|
||||
await sleep(60);
|
||||
// The max sample (60) dominates even after smaller ones land.
|
||||
expect(estimator.getLagMs()).toBe(60);
|
||||
});
|
||||
|
||||
it("widens immediately on an observed (tripwire) lag and clamps wild values", () => {
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => 5),
|
||||
defaultLagMs: 5,
|
||||
maxLagMs: 1_000,
|
||||
});
|
||||
estimator.noteObservedLagMs(250);
|
||||
expect(estimator.getLagMs()).toBe(250);
|
||||
estimator.noteObservedLagMs(99_999);
|
||||
expect(estimator.getLagMs()).toBe(1_000);
|
||||
});
|
||||
|
||||
it("an observed lag floors the estimate past the sample window (until its TTL)", async () => {
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => 0), // caught-up zeros, like vanilla PG between writes
|
||||
sampleIntervalMs: 10,
|
||||
windowMs: 30,
|
||||
defaultLagMs: 0,
|
||||
observedFloorTtlMs: 10_000,
|
||||
});
|
||||
estimator.touch();
|
||||
estimator.noteObservedLagMs(150);
|
||||
// Long past windowMs the zeros have flushed the observation out of the window,
|
||||
// but the floor still carries it.
|
||||
await sleep(80);
|
||||
expect(estimator.getLagMs()).toBe(150);
|
||||
});
|
||||
|
||||
it("stops sampling once idle and resumes on touch", async () => {
|
||||
let probes = 0;
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => {
|
||||
probes++;
|
||||
return 1;
|
||||
}),
|
||||
sampleIntervalMs: 10,
|
||||
idleAfterMs: 20,
|
||||
});
|
||||
estimator.touch();
|
||||
await sleep(80);
|
||||
const afterIdle = probes;
|
||||
expect(afterIdle).toBeGreaterThan(0);
|
||||
await sleep(40);
|
||||
// No new samples while idle...
|
||||
expect(probes).toBe(afterIdle);
|
||||
// ...and touching resumes immediately.
|
||||
estimator.touch();
|
||||
await sleep(15);
|
||||
expect(probes).toBeGreaterThan(afterIdle);
|
||||
});
|
||||
|
||||
it("survives a throwing source and keeps the last known value", async () => {
|
||||
let fail = false;
|
||||
estimator = new ReplicaLagEstimator({
|
||||
source: source(async () => {
|
||||
if (fail) throw new Error("source down");
|
||||
return 33;
|
||||
}),
|
||||
sampleIntervalMs: 10,
|
||||
windowMs: 30,
|
||||
defaultLagMs: 0,
|
||||
});
|
||||
estimator.touch();
|
||||
await sleep(25);
|
||||
expect(estimator.getLagMs()).toBe(33);
|
||||
fail = true;
|
||||
await sleep(60);
|
||||
// Window emptied, source failing — falls back to the last known sample.
|
||||
expect(estimator.getLagMs()).toBe(33);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FirstSupportedReplicaLagSource", () => {
|
||||
it("selects the first candidate whose sample succeeds and sticks with it", async () => {
|
||||
let auroraCalls = 0;
|
||||
let vanillaCalls = 0;
|
||||
const composed = new FirstSupportedReplicaLagSource([
|
||||
source(async () => {
|
||||
auroraCalls++;
|
||||
throw new Error("function aurora_replica_status() does not exist");
|
||||
}, "aurora"),
|
||||
source(async () => {
|
||||
vanillaCalls++;
|
||||
return 7;
|
||||
}, "vanilla-pg"),
|
||||
]);
|
||||
expect(composed.name).toBe("undetected");
|
||||
expect(await composed.sampleLagMs()).toBe(7);
|
||||
expect(composed.name).toBe("vanilla-pg");
|
||||
expect(await composed.sampleLagMs()).toBe(7);
|
||||
// The unsupported dialect was only probed during selection.
|
||||
expect(auroraCalls).toBe(1);
|
||||
expect(vanillaCalls).toBe(2);
|
||||
});
|
||||
|
||||
it("degrades to never-measuring when no candidate works", async () => {
|
||||
const composed = new FirstSupportedReplicaLagSource([
|
||||
source(async () => {
|
||||
throw new Error("nope");
|
||||
}),
|
||||
]);
|
||||
expect(await composed.sampleLagMs()).toBeUndefined();
|
||||
expect(await composed.sampleLagMs()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("a transient error after selection skips the sample without unselecting", async () => {
|
||||
let calls = 0;
|
||||
const composed = new FirstSupportedReplicaLagSource([
|
||||
source(async () => {
|
||||
calls++;
|
||||
if (calls === 2) throw new Error("transient");
|
||||
return 11;
|
||||
}, "flaky"),
|
||||
]);
|
||||
expect(await composed.sampleLagMs()).toBe(11);
|
||||
expect(await composed.sampleLagMs()).toBeUndefined(); // transient error -> skipped sample
|
||||
expect(await composed.sampleLagMs()).toBe(11); // still selected
|
||||
expect(composed.name).toBe("flaky");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { redisTest } from "@internal/testcontainers";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
type ChangeRecord,
|
||||
decodeChangeRecord,
|
||||
encodeChangeRecord,
|
||||
RunChangeNotifier,
|
||||
} from "~/services/realtime/runChangeNotifier.server";
|
||||
|
||||
function toRedisOptions(redisOptions: { host?: string; port?: number; password?: string }) {
|
||||
return {
|
||||
host: redisOptions.host,
|
||||
port: redisOptions.port,
|
||||
password: redisOptions.password,
|
||||
tlsDisabled: true,
|
||||
clusterMode: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Time for a SUBSCRIBE to register server-side before we publish.
|
||||
const SUBSCRIBE_SETTLE_MS = 250;
|
||||
|
||||
describe("RunChangeNotifier", () => {
|
||||
redisTest(
|
||||
"delivers a published change to an env subscriber",
|
||||
{ timeout: 30_000 },
|
||||
async ({ redisOptions }) => {
|
||||
const notifier = new RunChangeNotifier({ redis: toRedisOptions(redisOptions) });
|
||||
try {
|
||||
const received: ChangeRecord[] = [];
|
||||
const unsubscribe = notifier.subscribeToEnv("env_1", (records) =>
|
||||
received.push(...records)
|
||||
);
|
||||
expect(notifier.activeSubscriptionCount).toBe(1);
|
||||
|
||||
await sleep(SUBSCRIBE_SETTLE_MS);
|
||||
notifier.publish({ runId: "run_1", envId: "env_1", tags: ["a"], batchId: "batch_1" });
|
||||
|
||||
await vi.waitFor(() => expect(received.some((r) => r.runId === "run_1")).toBe(true), {
|
||||
timeout: 5_000,
|
||||
interval: 50,
|
||||
});
|
||||
const got = received.find((r) => r.runId === "run_1")!;
|
||||
expect(got.tags).toEqual(["a"]);
|
||||
expect(got.batchId).toBe("batch_1");
|
||||
|
||||
unsubscribe();
|
||||
// Cleanup is deferred until Redis confirms UNSUBSCRIBE, so the count converges to 0.
|
||||
await vi.waitFor(() => expect(notifier.activeSubscriptionCount).toBe(0), {
|
||||
timeout: 5_000,
|
||||
interval: 50,
|
||||
});
|
||||
} finally {
|
||||
await notifier.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
redisTest(
|
||||
"does not deliver a change for a different env",
|
||||
{ timeout: 30_000 },
|
||||
async ({ redisOptions }) => {
|
||||
const notifier = new RunChangeNotifier({ redis: toRedisOptions(redisOptions) });
|
||||
try {
|
||||
const received: ChangeRecord[] = [];
|
||||
notifier.subscribeToEnv("env_a", (records) => received.push(...records));
|
||||
|
||||
await sleep(SUBSCRIBE_SETTLE_MS);
|
||||
notifier.publish({ runId: "run_1", envId: "env_b", tags: [] }); // different env
|
||||
await sleep(500);
|
||||
|
||||
expect(received).toHaveLength(0);
|
||||
} finally {
|
||||
await notifier.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
redisTest(
|
||||
"coalesces a burst of env publishes into far fewer batches than publishes (lossless)",
|
||||
{ timeout: 30_000 },
|
||||
async ({ redisOptions }) => {
|
||||
const notifier = new RunChangeNotifier({
|
||||
redis: toRedisOptions(redisOptions),
|
||||
envWakeCoalesceWindowMs: 100,
|
||||
});
|
||||
try {
|
||||
let batches = 0;
|
||||
const runIds = new Set<string>();
|
||||
notifier.subscribeToEnv("env_burst", (records) => {
|
||||
batches++;
|
||||
for (const r of records) runIds.add(r.runId);
|
||||
});
|
||||
|
||||
await sleep(SUBSCRIBE_SETTLE_MS);
|
||||
let pubs = 0;
|
||||
const end = Date.now() + 1_000;
|
||||
while (Date.now() < end) {
|
||||
notifier.publish({ runId: `r${pubs++}`, envId: "env_burst", tags: [] });
|
||||
await sleep(5);
|
||||
}
|
||||
await sleep(300);
|
||||
|
||||
expect(pubs).toBeGreaterThan(100);
|
||||
expect(batches).toBeGreaterThanOrEqual(1);
|
||||
// Leading-edge throttle: far fewer deliveries than publishes...
|
||||
expect(batches).toBeLessThan(pubs / 4);
|
||||
// ...but lossless — the batch accumulates every run that changed in the window.
|
||||
expect(runIds.size).toBeGreaterThan(pubs / 2);
|
||||
} finally {
|
||||
await notifier.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Sharded pub/sub (SSUBSCRIBE/SPUBLISH/smessage) wiring — validated end to end on a
|
||||
// single node (Redis 7.2 accepts these and delivers same-node). Multi-shard ROUTING
|
||||
// needs a real cluster (the cluster fixture covers that); this proves the command path.
|
||||
redisTest(
|
||||
"delivers via sharded pub/sub on the env channel",
|
||||
{ timeout: 30_000 },
|
||||
async ({ redisOptions }) => {
|
||||
const notifier = new RunChangeNotifier({
|
||||
redis: toRedisOptions(redisOptions),
|
||||
shardedPubSub: true,
|
||||
});
|
||||
try {
|
||||
const received: ChangeRecord[] = [];
|
||||
notifier.subscribeToEnv("env_sharded", (records) => received.push(...records));
|
||||
|
||||
await sleep(SUBSCRIBE_SETTLE_MS);
|
||||
notifier.publish({ runId: "run_1", envId: "env_sharded", tags: ["a"] });
|
||||
|
||||
await vi.waitFor(() => expect(received.some((r) => r.runId === "run_1")).toBe(true), {
|
||||
timeout: 5_000,
|
||||
interval: 50,
|
||||
});
|
||||
} finally {
|
||||
await notifier.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
describe("ChangeRecord codec", () => {
|
||||
it("round-trips a full record (tags with a separator survive)", () => {
|
||||
const encoded = encodeChangeRecord({
|
||||
v: 1,
|
||||
runId: "run_1",
|
||||
envId: "env_1",
|
||||
tags: ["a", "b,c"],
|
||||
batchId: "batch_1",
|
||||
});
|
||||
expect(decodeChangeRecord(encoded)).toMatchObject({
|
||||
v: 1,
|
||||
runId: "run_1",
|
||||
envId: "env_1",
|
||||
tags: ["a", "b,c"],
|
||||
batchId: "batch_1",
|
||||
});
|
||||
});
|
||||
|
||||
it("decodes a bare runId to a partial record (tags undefined)", () => {
|
||||
// A bare/legacy frame: the consumer falls back to hydrate-to-classify.
|
||||
const decoded = decodeChangeRecord("run_3");
|
||||
expect(decoded.runId).toBe("run_3");
|
||||
expect(decoded.tags).toBeUndefined();
|
||||
});
|
||||
|
||||
it("falls back to a bare runId on an unparseable message", () => {
|
||||
expect(decodeChangeRecord("{not json").runId).toBe("{not json");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { PostgresRunStore } from "@internal/run-store";
|
||||
import { buildHydratorSelect, RunHydrator } from "~/services/realtime/runReader.server";
|
||||
|
||||
describe("buildHydratorSelect", () => {
|
||||
it("returns the full select when nothing is skipped", () => {
|
||||
const select = buildHydratorSelect([]);
|
||||
expect(select.id).toBe(true);
|
||||
expect(select.payload).toBe(true);
|
||||
expect(select.output).toBe(true);
|
||||
expect(select.metadata).toBe(true);
|
||||
expect(select.error).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps protocol-reserved columns even when asked to skip them", () => {
|
||||
// Reserved columns are always emitted by the serializer, so hydration must keep
|
||||
// them regardless of skipColumns or the output is null/incorrect.
|
||||
const select = buildHydratorSelect([
|
||||
"status",
|
||||
"taskIdentifier",
|
||||
"createdAt",
|
||||
"friendlyId",
|
||||
"payload",
|
||||
]);
|
||||
expect(select.status).toBe(true);
|
||||
expect(select.taskIdentifier).toBe(true);
|
||||
expect(select.createdAt).toBe(true);
|
||||
expect(select.friendlyId).toBe(true);
|
||||
// A non-reserved skipped column is still dropped.
|
||||
expect(select.payload).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops skipped columns but always keeps id + updatedAt", () => {
|
||||
const select = buildHydratorSelect(["payload", "output", "metadata", "error"]);
|
||||
expect(select.payload).toBeUndefined();
|
||||
expect(select.output).toBeUndefined();
|
||||
expect(select.metadata).toBeUndefined();
|
||||
expect(select.error).toBeUndefined();
|
||||
// Needed internally regardless of skipColumns (keys the row, drives the diff/offset).
|
||||
expect(select.id).toBe(true);
|
||||
expect(select.updatedAt).toBe(true);
|
||||
// A non-skipped column survives.
|
||||
expect(select.status).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RunHydrator.hydrateByIds column projection", () => {
|
||||
function makeHydrator() {
|
||||
let capturedSelect: Record<string, boolean> | undefined;
|
||||
const replica = {
|
||||
taskRun: {
|
||||
findMany: vi.fn(async ({ select }: { select: Record<string, boolean> }) => {
|
||||
capturedSelect = select;
|
||||
return [];
|
||||
}),
|
||||
},
|
||||
} as any;
|
||||
const runStore = new PostgresRunStore({ prisma: replica, readOnlyPrisma: replica });
|
||||
return { hydrator: new RunHydrator({ replica, runStore }), getSelect: () => capturedSelect };
|
||||
}
|
||||
|
||||
it("projects the SELECT by skipColumns", async () => {
|
||||
const { hydrator, getSelect } = makeHydrator();
|
||||
await hydrator.hydrateByIds("env_1", ["run_1"], ["payload", "output"]);
|
||||
const select = getSelect()!;
|
||||
expect(select.payload).toBeUndefined();
|
||||
expect(select.output).toBeUndefined();
|
||||
expect(select.id).toBe(true);
|
||||
expect(select.updatedAt).toBe(true);
|
||||
});
|
||||
|
||||
it("selects the full column set when no skipColumns are given", async () => {
|
||||
const { hydrator, getSelect } = makeHydrator();
|
||||
await hydrator.hydrateByIds("env_1", ["run_1"]);
|
||||
expect(getSelect()!.payload).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,532 @@
|
||||
import { heteroPostgresTest, postgresTest } from "@internal/testcontainers";
|
||||
import { PostgresRunStore } from "@internal/run-store";
|
||||
import type { ReadClient, RunStore } from "@internal/run-store";
|
||||
import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { Prisma, PrismaClient } from "@trigger.dev/database";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import { RunHydrator } from "~/services/realtime/runReader.server";
|
||||
|
||||
// Realtime read-route proof for the RunHydrator.
|
||||
//
|
||||
// On origin/main the realtime RunHydrator's two run-ops reads already flow through the runStore
|
||||
// seam: `hydrateByIds` -> `runStore.findRuns(..., replica)` and `#fetch` -> `runStore.findRun(...,
|
||||
// replica)`. The split-aware routing (new-DB-first, legacy READ REPLICA only for ids not
|
||||
// known-migrated) is the store's job below the seam, so this file proves the hydrator *inherits*
|
||||
// that routing — plus that the single-flight + short-TTL cache and the skipColumns projection
|
||||
// (which live in the hydrator, not the store) are unaffected by the seam.
|
||||
//
|
||||
// The heterogeneous fixture gives real legacy + new Postgres containers; NO DB is mocked. The ONLY
|
||||
// non-DB fake is the residency selector that the routing-shaped store uses (`ownerEngine`: run-ops id ->
|
||||
// NEW, cuid -> LEGACY), exactly the substrate the RoutingRunStore ships. Run ids are 25 chars (cuid
|
||||
// -> LEGACY) or v1-shaped (26 chars, version "1" at index 25 -> NEW) so the classifier routes them deterministically.
|
||||
|
||||
// 25-char internal id -> cuid -> LEGACY; v1 internal id (26 chars, version "1" at index 25) -> NEW. The
|
||||
// classifier strips a leading `<prefix>_`, so these ids must carry NO underscore (a bare
|
||||
// alphanumeric body of the exact length).
|
||||
function newId(label: string): string {
|
||||
return ("k" + label.replace(/[^0-9a-v]/g, "")).padEnd(24, "0").slice(0, 24) + "01";
|
||||
}
|
||||
function legacyId(label: string): string {
|
||||
return ("c" + label.replace(/[^a-z0-9]/gi, "")).padEnd(25, "0").slice(0, 25);
|
||||
}
|
||||
|
||||
async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${slugSuffix}`,
|
||||
slug: `project-${slugSuffix}`,
|
||||
externalRef: `proj_${slugSuffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${slugSuffix}`,
|
||||
pkApiKey: `pk_dev_${slugSuffix}`,
|
||||
shortcode: `short_${slugSuffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
async function seedRun(
|
||||
prisma: PrismaClient,
|
||||
params: {
|
||||
runId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
payload?: string;
|
||||
output?: string | null;
|
||||
metadata?: string | null;
|
||||
runTags?: string[];
|
||||
error?: Prisma.InputJsonValue;
|
||||
}
|
||||
) {
|
||||
await prisma.taskRun.create({
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: `run_friendly_${params.runId.slice(0, 8)}`,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: params.payload ?? '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
...(params.output !== undefined && { output: params.output }),
|
||||
outputType: "application/json",
|
||||
...(params.metadata !== undefined && { metadata: params.metadata }),
|
||||
...(params.error !== undefined && { error: params.error }),
|
||||
traceContext: {},
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
runTags: params.runTags ?? ["alpha", "beta"],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A routing-shaped RunStore: routes the single-run `findRun` by residency (the exact substrate
|
||||
* the RoutingRunStore ships) and fans `findRuns` out across NEW + LEGACY, merging by id
|
||||
* (the union/dedup the routing store owns; this hydrator inherits it). For not-known-migrated ids
|
||||
* the read falls back to the LEGACY slot — which is wired over a READ REPLICA handle, never a
|
||||
* writer. Only `findRun`/`findRuns` (the two reads this unit exercises) are implemented; the rest
|
||||
* throw so any accidental call surfaces. The only non-DB fake here is the residency selector.
|
||||
*
|
||||
* By design the router ignores the explicit read `client` and reads off the selected slot's OWN
|
||||
* configured replica, so the hydrator's `replica` arg is dropped here.
|
||||
*/
|
||||
function makeRoutingShapedStore(options: {
|
||||
newStore: PostgresRunStore;
|
||||
legacyStore: PostgresRunStore;
|
||||
classify?: (id: string) => Residency;
|
||||
}): RunStore {
|
||||
const classify = options.classify ?? ownerEngine;
|
||||
const route = (id: string | undefined): PostgresRunStore => {
|
||||
if (typeof id !== "string") return options.legacyStore;
|
||||
try {
|
||||
return classify(id) === "NEW" ? options.newStore : options.legacyStore;
|
||||
} catch {
|
||||
// Not known-migrated / unclassifiable -> fall back to the LEGACY read replica only.
|
||||
return options.legacyStore;
|
||||
}
|
||||
};
|
||||
|
||||
const idFromWhere = (where: Prisma.TaskRunWhereInput): string | undefined => {
|
||||
const id = where.id;
|
||||
if (typeof id === "string") return id;
|
||||
if (id && typeof id === "object" && "equals" in id && typeof id.equals === "string") {
|
||||
return id.equals;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const handler: ProxyHandler<RunStore> = {
|
||||
get(_target, prop) {
|
||||
if (prop === "findRun") {
|
||||
// Drop the explicit `client`: the selected slot reads off its OWN replica.
|
||||
return (where: Prisma.TaskRunWhereInput, args: unknown, _client?: ReadClient) =>
|
||||
(route(idFromWhere(where)).findRun as (...rest: unknown[]) => Promise<unknown>)(
|
||||
where,
|
||||
args
|
||||
);
|
||||
}
|
||||
if (prop === "findRuns") {
|
||||
return async (
|
||||
args: { where: Prisma.TaskRunWhereInput; select: Prisma.TaskRunSelect },
|
||||
_client?: ReadClient
|
||||
) => {
|
||||
// Fan out across both slots (each on its OWN replica) and merge by id (the routing
|
||||
// store's union/dedup contract).
|
||||
const [fromNew, fromLegacy] = await Promise.all([
|
||||
options.newStore.findRuns(args as never),
|
||||
options.legacyStore.findRuns(args as never),
|
||||
]);
|
||||
const byId = new Map<string, Record<string, unknown>>();
|
||||
for (const row of [...fromLegacy, ...fromNew] as Record<string, unknown>[]) {
|
||||
byId.set(row.id as string, row);
|
||||
}
|
||||
return [...byId.values()];
|
||||
};
|
||||
}
|
||||
throw new Error(`routing-shaped store: ${String(prop)} not implemented in test`);
|
||||
},
|
||||
};
|
||||
|
||||
return new Proxy({} as RunStore, handler);
|
||||
}
|
||||
|
||||
describe("RunHydrator read-route through the runStore seam (legacy + new)", () => {
|
||||
// Realtime hydrate pulls run-ops rows from the run-ops replica. A split hydrate returns the
|
||||
// union of NEW + LEGACY-replica rows, byte-identical to source, via both
|
||||
// getRunById and hydrateByIds.
|
||||
heteroPostgresTest(
|
||||
"split hydrate returns the NEW + legacy-replica union, byte-identical",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
|
||||
const seed14 = await seedEnvironment(prisma14, "u14");
|
||||
const seed17 = await seedEnvironment(prisma17, "u17");
|
||||
// Both seed envs use the SAME runtimeEnvironmentId so the env-scoped `where` matches across
|
||||
// the two physical DBs (each env row is local to its DB but carries the same id).
|
||||
const envId = seed17.environment.id;
|
||||
await prisma14.runtimeEnvironment.update({
|
||||
where: { id: seed14.environment.id },
|
||||
data: { id: envId },
|
||||
});
|
||||
|
||||
const newRunId = newId("union_new");
|
||||
const legacyRunId = legacyId("union_old");
|
||||
|
||||
await seedRun(prisma17, {
|
||||
runId: newRunId,
|
||||
organizationId: seed17.organization.id,
|
||||
projectId: seed17.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: '{"side":"new"}',
|
||||
output: '{"result":42}',
|
||||
metadata: '{"m":1}',
|
||||
runTags: ["new", "z"],
|
||||
error: { type: "BUILT_IN_ERROR", name: "Boom", message: "new-side" },
|
||||
});
|
||||
await seedRun(prisma14, {
|
||||
runId: legacyRunId,
|
||||
organizationId: seed14.organization.id,
|
||||
projectId: seed14.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: '{"side":"legacy"}',
|
||||
output: null,
|
||||
metadata: null,
|
||||
runTags: ["legacy", "a"],
|
||||
error: { type: "STRING_ERROR", raw: "legacy-side" },
|
||||
});
|
||||
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore });
|
||||
|
||||
const rows = await hydrator.hydrateByIds(envId, [newRunId, legacyRunId]);
|
||||
expect(rows.map((r) => r.id).sort()).toEqual([legacyRunId, newRunId].sort());
|
||||
|
||||
const newRow = rows.find((r) => r.id === newRunId)!;
|
||||
const legacyRow = rows.find((r) => r.id === legacyRunId)!;
|
||||
|
||||
// Byte-identical to source incl. JSON columns, runTags, error JSON.
|
||||
expect(newRow.payload).toBe('{"side":"new"}');
|
||||
expect(newRow.output).toBe('{"result":42}');
|
||||
expect(newRow.metadata).toBe('{"m":1}');
|
||||
expect(newRow.runTags).toEqual(["new", "z"]);
|
||||
expect(newRow.error).toEqual({ type: "BUILT_IN_ERROR", name: "Boom", message: "new-side" });
|
||||
|
||||
expect(legacyRow.payload).toBe('{"side":"legacy"}');
|
||||
expect(legacyRow.output).toBeNull();
|
||||
expect(legacyRow.metadata).toBeNull();
|
||||
expect(legacyRow.runTags).toEqual(["legacy", "a"]);
|
||||
expect(legacyRow.error).toEqual({ type: "STRING_ERROR", raw: "legacy-side" });
|
||||
|
||||
// getRunById resolves each individual run from its correct source through the seam.
|
||||
const newById = await hydrator.getRunById(envId, newRunId);
|
||||
const legacyById = await hydrator.getRunById(envId, legacyRunId);
|
||||
expect(newById?.payload).toBe('{"side":"new"}');
|
||||
expect(legacyById?.payload).toBe('{"side":"legacy"}');
|
||||
}
|
||||
);
|
||||
|
||||
// A known-migrated (NEW-residency) run is NOT re-probed on the legacy replica.
|
||||
heteroPostgresTest(
|
||||
"known-migrated run is never probed on the legacy slot",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
const legacyFindRunSpy = vi.spyOn(legacyStore, "findRun");
|
||||
|
||||
const seed17 = await seedEnvironment(prisma17, "k17");
|
||||
const envId = seed17.environment.id;
|
||||
const migratedRunId = newId("known_mig");
|
||||
await seedRun(prisma17, {
|
||||
runId: migratedRunId,
|
||||
organizationId: seed17.organization.id,
|
||||
projectId: seed17.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
});
|
||||
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore });
|
||||
|
||||
const row = await hydrator.getRunById(envId, migratedRunId);
|
||||
expect(row?.id).toBe(migratedRunId);
|
||||
// The NEW-residency id resolved against the NEW slot only — the legacy probe never ran.
|
||||
expect(legacyFindRunSpy).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
|
||||
// An old in-retention run is served from the LEGACY read replica (never a writer/primary path).
|
||||
heteroPostgresTest(
|
||||
"old in-retention run served from the legacy replica slot",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
// The LEGACY slot exposes only a read/replica handle: `prisma14` is wired as BOTH prisma and
|
||||
// readOnlyPrisma, and the hydrator passes it as the explicit read client — there is no
|
||||
// legacy-writer read path on the read route (the replica-only invariant is structural in the
|
||||
// store; asserted here as inheritance).
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
|
||||
const seed14 = await seedEnvironment(prisma14, "o14");
|
||||
const envId = seed14.environment.id;
|
||||
const oldRunId = legacyId("old_run");
|
||||
await seedRun(prisma14, {
|
||||
runId: oldRunId,
|
||||
organizationId: seed14.organization.id,
|
||||
projectId: seed14.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: '{"era":"old"}',
|
||||
});
|
||||
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore });
|
||||
|
||||
const byId = await hydrator.getRunById(envId, oldRunId);
|
||||
expect(byId?.payload).toBe('{"era":"old"}');
|
||||
|
||||
const [hydrated] = await hydrator.hydrateByIds(envId, [oldRunId]);
|
||||
expect(hydrated.payload).toBe('{"era":"old"}');
|
||||
}
|
||||
);
|
||||
|
||||
// Terminal-metadata read-seam: a NEW-resident (run-ops id) run's final metadata is hydrated through
|
||||
// the owning (NEW) store, not off a generic legacy replica. Asserts read-seam ROUTING for the
|
||||
// terminal read; it is not a hard ordering/consistency guarantee about when the terminal marker
|
||||
// and the row's terminal columns converge.
|
||||
heteroPostgresTest(
|
||||
"terminal hydrate reads a NEW-resident run's final metadata through the owning store",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
const legacyFindRunSpy = vi.spyOn(legacyStore, "findRun");
|
||||
|
||||
const seed17 = await seedEnvironment(prisma17, "term17");
|
||||
const envId = seed17.environment.id;
|
||||
const terminalRunId = newId("terminal_run");
|
||||
|
||||
// A terminal run with its final metadata persisted on the NEW store only.
|
||||
await seedRun(prisma17, {
|
||||
runId: terminalRunId,
|
||||
organizationId: seed17.organization.id,
|
||||
projectId: seed17.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
output: '{"result":"final"}',
|
||||
metadata: '{"done":true}',
|
||||
});
|
||||
|
||||
// A generic legacy replica would miss the NEW row entirely — the metadata must come off NEW.
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 0 });
|
||||
|
||||
const snapshot = await hydrator.getRunById(envId, terminalRunId);
|
||||
expect(snapshot?.id).toBe(terminalRunId);
|
||||
expect(snapshot?.metadata).toBe('{"done":true}');
|
||||
expect(snapshot?.output).toBe('{"result":"final"}');
|
||||
// The NEW-residency terminal read never touched the legacy slot.
|
||||
expect(legacyFindRunSpy).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
|
||||
// A live-migrated run continues streaming across the seam crossing with no gap.
|
||||
heteroPostgresTest(
|
||||
"live-migrated run continues streaming across the seam crossing",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
|
||||
const seed14 = await seedEnvironment(prisma14, "m14");
|
||||
const seed17 = await seedEnvironment(prisma17, "m17");
|
||||
const envId = seed17.environment.id;
|
||||
await prisma14.runtimeEnvironment.update({
|
||||
where: { id: seed14.environment.id },
|
||||
data: { id: envId },
|
||||
});
|
||||
|
||||
// The run starts life on LEGACY; the residency selector classifies it NEW once it migrates.
|
||||
// We model the migration by seeding the same run id on LEGACY first, then on NEW, while
|
||||
// flipping the classifier from LEGACY to NEW for that id at the seam crossing.
|
||||
const runId = legacyId("migrating");
|
||||
await seedRun(prisma14, {
|
||||
runId,
|
||||
organizationId: seed14.organization.id,
|
||||
projectId: seed14.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: '{"home":"legacy"}',
|
||||
});
|
||||
|
||||
let migrated = false;
|
||||
const classify = (id: string): Residency =>
|
||||
id === runId && migrated ? "NEW" : ownerEngine(id);
|
||||
const legacyFindRunSpy = vi.spyOn(legacyStore, "findRun");
|
||||
|
||||
// Use a 0ms TTL so each getRunById re-reads through the seam (no cached stale row across the
|
||||
// crossing). Single-flight/TTL are proven separately below.
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore, classify });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 0 });
|
||||
|
||||
// Before migration: served from LEGACY.
|
||||
const before = await hydrator.getRunById(envId, runId);
|
||||
expect(before?.payload).toBe('{"home":"legacy"}');
|
||||
expect(legacyFindRunSpy).toHaveBeenCalled();
|
||||
|
||||
// Migrate: the run now lives on NEW and the classifier routes it NEW.
|
||||
await seedRun(prisma17, {
|
||||
runId,
|
||||
organizationId: seed17.organization.id,
|
||||
projectId: seed17.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: '{"home":"new"}',
|
||||
});
|
||||
migrated = true;
|
||||
legacyFindRunSpy.mockClear();
|
||||
|
||||
// After migration: served from NEW, with no gap and no legacy re-probe.
|
||||
const after = await hydrator.getRunById(envId, runId);
|
||||
expect(after?.payload).toBe('{"home":"new"}');
|
||||
expect(after?.id).toBe(runId);
|
||||
expect(legacyFindRunSpy).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("RunHydrator single-flight + TTL cache intact across the seam", () => {
|
||||
// The cache/single-flight live in the hydrator, independent of the storage seam. Proven in
|
||||
// SPLIT mode here (a counting wrapper over the selected underlying store's read).
|
||||
heteroPostgresTest(
|
||||
"split mode: two concurrent getRunById -> one underlying read; repeat within TTL is cached",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
const newFindRunSpy = vi.spyOn(newStore, "findRun");
|
||||
|
||||
const seed17 = await seedEnvironment(prisma17, "s17");
|
||||
const envId = seed17.environment.id;
|
||||
const runId = newId("cached_run");
|
||||
await seedRun(prisma17, {
|
||||
runId,
|
||||
organizationId: seed17.organization.id,
|
||||
projectId: seed17.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
});
|
||||
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 60_000 });
|
||||
|
||||
// Two concurrent calls -> single-flight collapses to ONE underlying read.
|
||||
const [a, b] = await Promise.all([
|
||||
hydrator.getRunById(envId, runId),
|
||||
hydrator.getRunById(envId, runId),
|
||||
]);
|
||||
expect(a?.id).toBe(runId);
|
||||
expect(b?.id).toBe(runId);
|
||||
expect(newFindRunSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// A third call within the TTL returns the cached value with no new read.
|
||||
const c = await hydrator.getRunById(envId, runId);
|
||||
expect(c?.id).toBe(runId);
|
||||
expect(newFindRunSpy).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
);
|
||||
|
||||
// A cached `null` (missing run) is a valid not-found hit and is not re-read within the TTL.
|
||||
heteroPostgresTest(
|
||||
"split mode: a cached null (missing run) is not re-read within the TTL",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 });
|
||||
const newFindRunSpy = vi.spyOn(newStore, "findRun");
|
||||
|
||||
const seed17 = await seedEnvironment(prisma17, "n17");
|
||||
const envId = seed17.environment.id;
|
||||
const missingRunId = newId("missing_run");
|
||||
|
||||
const runStore = makeRoutingShapedStore({ newStore, legacyStore });
|
||||
const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 60_000 });
|
||||
|
||||
const first = await hydrator.getRunById(envId, missingRunId);
|
||||
expect(first).toBeNull();
|
||||
expect(newFindRunSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
const second = await hydrator.getRunById(envId, missingRunId);
|
||||
expect(second).toBeNull();
|
||||
// Still one read — the null was cached as a valid "not found" hit.
|
||||
expect(newFindRunSpy).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("RunHydrator single-DB passthrough (one PostgresRunStore over one client)", () => {
|
||||
// Passthrough: in single-DB the store is one PostgresRunStore over one client; the hydrator
|
||||
// behaves byte-for-byte as today. No split branch, no legacy slot, no second connection.
|
||||
postgresTest(
|
||||
"single store: getRunById + hydrateByIds read from the one client, cache intact",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma }) => {
|
||||
const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma });
|
||||
const findRunSpy = vi.spyOn(store, "findRun");
|
||||
|
||||
const seed = await seedEnvironment(prisma, "sd1");
|
||||
const envId = seed.environment.id;
|
||||
const runIdA = newId("single_a");
|
||||
const runIdB = legacyId("single_b");
|
||||
for (const runId of [runIdA, runIdB]) {
|
||||
await seedRun(prisma, {
|
||||
runId,
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: envId,
|
||||
payload: `{"id":"${runId}"}`,
|
||||
});
|
||||
}
|
||||
|
||||
const hydrator = new RunHydrator({ replica: prisma, runStore: store, cacheTtlMs: 60_000 });
|
||||
|
||||
// hydrateByIds returns both rows from the single client.
|
||||
const rows = await hydrator.hydrateByIds(envId, [runIdA, runIdB]);
|
||||
expect(rows.map((r) => r.id).sort()).toEqual([runIdA, runIdB].sort());
|
||||
|
||||
// getRunById hydrates from the single store; the cache short-circuits a repeat read.
|
||||
const a1 = await hydrator.getRunById(envId, runIdA);
|
||||
const a2 = await hydrator.getRunById(envId, runIdA);
|
||||
expect(a1?.payload).toBe(`{"id":"${runIdA}"}`);
|
||||
expect(a2?.payload).toBe(`{"id":"${runIdA}"}`);
|
||||
expect(findRunSpy).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
);
|
||||
|
||||
// Empty id-set short-circuits with no store call.
|
||||
postgresTest("empty id-set returns [] without touching the store", async ({ prisma }) => {
|
||||
const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma });
|
||||
const findRunsSpy = vi.spyOn(store, "findRuns");
|
||||
const hydrator = new RunHydrator({ replica: prisma, runStore: store });
|
||||
|
||||
const rows = await hydrator.hydrateByIds("env_none", []);
|
||||
expect(rows).toEqual([]);
|
||||
expect(findRunsSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,223 @@
|
||||
import {
|
||||
type RealtimeRunRow,
|
||||
serializeRunRow,
|
||||
} from "~/services/realtime/electricStreamProtocol.server";
|
||||
import { type RunListFilter } from "~/services/realtime/runReader.server";
|
||||
import { RealtimeShadowComparator } from "~/services/realtime/shadowCompare.server";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
function sampleRow(overrides: Partial<RealtimeRunRow> = {}): RealtimeRunRow {
|
||||
return {
|
||||
id: "run_a",
|
||||
taskIdentifier: "my-task",
|
||||
createdAt: new Date("2026-06-07T09:00:00.000Z"),
|
||||
updatedAt: new Date("2026-06-07T10:05:30.123Z"),
|
||||
startedAt: null,
|
||||
delayUntil: null,
|
||||
queuedAt: null,
|
||||
expiredAt: null,
|
||||
completedAt: null,
|
||||
friendlyId: "run_friendly_a",
|
||||
number: 7,
|
||||
isTest: true,
|
||||
status: "EXECUTING",
|
||||
usageDurationMs: 1234,
|
||||
costInCents: 0.55,
|
||||
baseCostInCents: 0.25,
|
||||
ttl: "1h",
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
metadata: null,
|
||||
metadataType: "application/json",
|
||||
output: null,
|
||||
outputType: "application/json",
|
||||
runTags: ["a", "b"],
|
||||
error: null,
|
||||
realtimeStreams: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const UP_TO_DATE = { headers: { control: "up-to-date" } };
|
||||
|
||||
function insert(value: Record<string, string | null>) {
|
||||
return { key: `"public"."TaskRun"/"${value.id}"`, value, headers: { operation: "insert" } };
|
||||
}
|
||||
|
||||
function makeComparator(
|
||||
rowsById: Record<string, RealtimeRunRow | null>,
|
||||
resolvedIds: string[] = []
|
||||
) {
|
||||
return new RealtimeShadowComparator({
|
||||
runReader: {
|
||||
getRunById: async (_env: string, id: string) => rowsById[id] ?? null,
|
||||
hydrateByIds: async (_env: string, ids: string[]) =>
|
||||
ids.map((id) => rowsById[id]).filter((row): row is RealtimeRunRow => Boolean(row)),
|
||||
} as any,
|
||||
runListResolver: { resolveMatchingRunIds: async (_f: RunListFilter) => resolvedIds } as any,
|
||||
});
|
||||
}
|
||||
|
||||
describe("RealtimeShadowComparator serialization", () => {
|
||||
it("counts a faithful re-serialization as a match", async () => {
|
||||
const row = sampleRow();
|
||||
const body = JSON.stringify([insert(serializeRunRow(row)), UP_TO_DATE]);
|
||||
const cmp = makeComparator({ run_a: row });
|
||||
|
||||
const out = await cmp.compare({
|
||||
feed: "run",
|
||||
electricBody: body,
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
});
|
||||
|
||||
expect(out.serializationMatched).toBe(1);
|
||||
expect(out.serializationDiverged).toBe(0);
|
||||
expect(out.serializationSkew).toBe(0);
|
||||
expect(out.diffs).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not flag semantically-equivalent but differently-encoded values", async () => {
|
||||
const row = sampleRow();
|
||||
// Electric encodes bool as "true" (native uses "t"), a number with a trailing
|
||||
// zero, and a timestamp without millis — all equal after decoding.
|
||||
const value = {
|
||||
...serializeRunRow(row),
|
||||
isTest: "true",
|
||||
costInCents: "0.5500",
|
||||
createdAt: "2026-06-07T09:00:00",
|
||||
};
|
||||
const body = JSON.stringify([insert(value), UP_TO_DATE]);
|
||||
const cmp = makeComparator({ run_a: row });
|
||||
|
||||
const out = await cmp.compare({
|
||||
feed: "run",
|
||||
electricBody: body,
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
});
|
||||
|
||||
expect(out.serializationMatched).toBe(1);
|
||||
expect(out.serializationDiverged).toBe(0);
|
||||
});
|
||||
|
||||
it("flags a genuine column divergence (same version)", async () => {
|
||||
const row = sampleRow();
|
||||
const value = { ...serializeRunRow(row), payload: '{"hello":"TAMPERED"}' };
|
||||
const body = JSON.stringify([insert(value), UP_TO_DATE]);
|
||||
const cmp = makeComparator({ run_a: row });
|
||||
|
||||
const out = await cmp.compare({
|
||||
feed: "run",
|
||||
electricBody: body,
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
});
|
||||
|
||||
expect(out.serializationDiverged).toBe(1);
|
||||
expect(out.serializationMatched).toBe(0);
|
||||
expect(out.diffs).toEqual([
|
||||
{
|
||||
runId: "run_a",
|
||||
column: "payload",
|
||||
electric: '{"hello":"TAMPERED"}',
|
||||
native: '{"hello":"world"}',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats DEQUEUED/EXECUTING as equivalent (legacy status rewrite)", async () => {
|
||||
const row = sampleRow({ status: "EXECUTING" });
|
||||
const value = { ...serializeRunRow(row), status: "DEQUEUED" };
|
||||
const body = JSON.stringify([insert(value), UP_TO_DATE]);
|
||||
const cmp = makeComparator({ run_a: row });
|
||||
|
||||
const out = await cmp.compare({
|
||||
feed: "run",
|
||||
electricBody: body,
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
});
|
||||
|
||||
expect(out.serializationDiverged).toBe(0);
|
||||
expect(out.serializationMatched).toBe(1);
|
||||
});
|
||||
|
||||
it("records skew when the row advanced between emit and refetch", async () => {
|
||||
const row = sampleRow();
|
||||
// Electric emitted an older version; the refetched row is newer.
|
||||
const value = {
|
||||
...serializeRunRow(sampleRow({ updatedAt: new Date("2026-06-07T10:00:00.000Z") })),
|
||||
};
|
||||
const body = JSON.stringify([insert(value), UP_TO_DATE]);
|
||||
const cmp = makeComparator({ run_a: row });
|
||||
|
||||
const out = await cmp.compare({
|
||||
feed: "run",
|
||||
electricBody: body,
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
});
|
||||
|
||||
expect(out.serializationSkew).toBe(1);
|
||||
expect(out.serializationMatched).toBe(0);
|
||||
expect(out.serializationDiverged).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RealtimeShadowComparator membership", () => {
|
||||
const filter: RunListFilter = {
|
||||
organizationId: "org_1",
|
||||
projectId: "proj_1",
|
||||
environmentId: "env_1",
|
||||
tags: ["t"],
|
||||
createdAtAfter: new Date("2026-06-06T00:00:00.000Z"),
|
||||
limit: 1000,
|
||||
};
|
||||
|
||||
function bodyFor(ids: string[]) {
|
||||
const msgs = ids.map((id) => insert(serializeRunRow(sampleRow({ id }))));
|
||||
return JSON.stringify([...msgs, UP_TO_DATE]);
|
||||
}
|
||||
|
||||
it("matches when Electric's set equals the native resolver's set", async () => {
|
||||
const cmp = makeComparator({ a: sampleRow({ id: "a" }), b: sampleRow({ id: "b" }) }, [
|
||||
"a",
|
||||
"b",
|
||||
]);
|
||||
const out = await cmp.compare({
|
||||
feed: "runs",
|
||||
electricBody: bodyFor(["a", "b"]),
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
membershipFilter: filter,
|
||||
});
|
||||
expect(out.membershipMatch).toBe(true);
|
||||
expect(out.missingInNative).toEqual([]);
|
||||
expect(out.extraInNative).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports rows missing from / extra in the native resolution", async () => {
|
||||
const cmp = makeComparator(
|
||||
{ a: sampleRow({ id: "a" }), b: sampleRow({ id: "b" }) },
|
||||
["a", "c"] // native missing b, has extra c
|
||||
);
|
||||
const out = await cmp.compare({
|
||||
feed: "runs",
|
||||
electricBody: bodyFor(["a", "b"]),
|
||||
environment: { id: "env_1" },
|
||||
skipColumns: [],
|
||||
isInitialSnapshot: true,
|
||||
membershipFilter: filter,
|
||||
});
|
||||
expect(out.membershipMatch).toBe(false);
|
||||
expect(out.missingInNative).toEqual(["b"]);
|
||||
expect(out.extraInNative).toEqual(["c"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,240 @@
|
||||
import { heteroPostgresTest, redisTest } from "@internal/testcontainers";
|
||||
import { PostgresRunStore } from "@internal/run-store";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import Redis from "ioredis";
|
||||
import { describe, expect } from "vitest";
|
||||
import { RedisRealtimeStreams } from "~/services/realtime/redisRealtimeStreams.server.js";
|
||||
|
||||
// Seeds organization -> project -> runtimeEnvironment -> taskRun on the given prisma client.
|
||||
// Mirrors the route's target run: a V2 run with an (optionally completed) lifecycle and an
|
||||
// initially-empty realtimeStreams array.
|
||||
async function seedRun(
|
||||
prisma: PrismaClient,
|
||||
params: {
|
||||
runId: string;
|
||||
slugSuffix: string;
|
||||
completedAt?: Date;
|
||||
}
|
||||
) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Test Organization",
|
||||
slug: `test-organization-${params.slugSuffix}`,
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Test Project",
|
||||
slug: `test-project-${params.slugSuffix}`,
|
||||
externalRef: `proj_${params.slugSuffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_apikey_${params.slugSuffix}`,
|
||||
pkApiKey: `pk_dev_apikey_${params.slugSuffix}`,
|
||||
shortcode: `short_code_${params.slugSuffix}`,
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.taskRun.create({
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: `run_friendly_${params.slugSuffix}`,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: organization.id,
|
||||
projectId: project.id,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
...(params.completedAt !== undefined && { completedAt: params.completedAt }),
|
||||
},
|
||||
});
|
||||
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
// The exact routed sequence performed by realtime.v1.streams.$runId.$target.$streamId(.append) PUT:
|
||||
// read the target via the store, then push the streamId iff it is not already present and the run
|
||||
// is not completed. Driving this against the store is the routed seam (no engine instance required).
|
||||
async function routedRegisterStream(
|
||||
store: PostgresRunStore,
|
||||
client: PrismaClient,
|
||||
runId: string,
|
||||
streamId: string
|
||||
): Promise<{ pushed: boolean }> {
|
||||
const target = await store.findRun(
|
||||
{ id: runId },
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
realtimeStreams: true,
|
||||
realtimeStreamsVersion: true,
|
||||
completedAt: true,
|
||||
},
|
||||
},
|
||||
client
|
||||
);
|
||||
|
||||
if (!target) {
|
||||
throw new Error("Run not found");
|
||||
}
|
||||
|
||||
// Completed-run guard (route returns 400 here).
|
||||
if (target.completedAt) {
|
||||
return { pushed: false };
|
||||
}
|
||||
|
||||
if (!target.realtimeStreams.includes(streamId)) {
|
||||
await store.pushRealtimeStream(target.id, streamId, client);
|
||||
return { pushed: true };
|
||||
}
|
||||
|
||||
return { pushed: false };
|
||||
}
|
||||
|
||||
describe("realtime stream registration — run-ops store routed writes", () => {
|
||||
heteroPostgresTest(
|
||||
"push routes to run-ops store for a run on the new DB",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma17, prisma14 }) => {
|
||||
// The run-ops store owns the PG17 (new) DB.
|
||||
const store = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
|
||||
const runId = "run_routed_push_new_db";
|
||||
await seedRun(prisma17, { runId, slugSuffix: "push17" });
|
||||
|
||||
const streamId = "stream-abc";
|
||||
const result = await routedRegisterStream(store, prisma17, runId, streamId);
|
||||
|
||||
expect(result.pushed).toBe(true);
|
||||
|
||||
// Write landed on the new (PG17) DB.
|
||||
const onNewDb = await prisma17.taskRun.findFirst({
|
||||
where: { id: runId },
|
||||
select: { realtimeStreams: true },
|
||||
});
|
||||
expect(onNewDb?.realtimeStreams).toContain(streamId);
|
||||
|
||||
// Write is isolated to the new DB — the legacy (PG14) DB carries no run with that streamId.
|
||||
const onLegacyDb = await prisma14.taskRun.findFirst({
|
||||
where: { realtimeStreams: { has: streamId } },
|
||||
select: { id: true },
|
||||
});
|
||||
expect(onLegacyDb).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"idempotent — already-registered streamId issues no second write",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma17 }) => {
|
||||
const store = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
|
||||
const runId = "run_routed_push_idempotent";
|
||||
await seedRun(prisma17, { runId, slugSuffix: "idem17" });
|
||||
|
||||
const streamId = "stream-once";
|
||||
|
||||
const first = await routedRegisterStream(store, prisma17, runId, streamId);
|
||||
expect(first.pushed).toBe(true);
|
||||
|
||||
const second = await routedRegisterStream(store, prisma17, runId, streamId);
|
||||
// The includes() guard skipped the second push.
|
||||
expect(second.pushed).toBe(false);
|
||||
|
||||
const row = await prisma17.taskRun.findFirst({
|
||||
where: { id: runId },
|
||||
select: { realtimeStreams: true },
|
||||
});
|
||||
// Exactly one entry — no duplicate appended.
|
||||
expect(row?.realtimeStreams).toEqual([streamId]);
|
||||
expect(row?.realtimeStreams).toHaveLength(1);
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"completed run guard issues no push",
|
||||
{ timeout: 60_000 },
|
||||
async ({ prisma17 }) => {
|
||||
const store = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 });
|
||||
|
||||
const runId = "run_routed_push_completed";
|
||||
await seedRun(prisma17, {
|
||||
runId,
|
||||
slugSuffix: "completed17",
|
||||
completedAt: new Date("2026-06-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
const streamId = "stream-late";
|
||||
const result = await routedRegisterStream(store, prisma17, runId, streamId);
|
||||
|
||||
// The completedAt guard blocks the push (route returns 400).
|
||||
expect(result.pushed).toBe(false);
|
||||
|
||||
const row = await prisma17.taskRun.findFirst({
|
||||
where: { id: runId },
|
||||
select: { realtimeStreams: true },
|
||||
});
|
||||
expect(row?.realtimeStreams).toEqual([]);
|
||||
}
|
||||
);
|
||||
|
||||
redisTest(
|
||||
"chunks flow — stream attaches and chunks are ingested",
|
||||
{ timeout: 30_000 },
|
||||
async ({ redisOptions }) => {
|
||||
const redis = new Redis(redisOptions);
|
||||
const streams = new RedisRealtimeStreams({ redis: redisOptions });
|
||||
|
||||
const runId = "run_chunks_flow";
|
||||
const streamId = "registered-stream";
|
||||
|
||||
const chunks = [
|
||||
JSON.stringify({ chunk: 0, data: "chunk 0" }),
|
||||
JSON.stringify({ chunk: 1, data: "chunk 1" }),
|
||||
JSON.stringify({ chunk: 2, data: "chunk 2" }),
|
||||
];
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) {
|
||||
controller.enqueue(encoder.encode(chunk + "\n"));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
const response = await streams.ingestData(stream, runId, streamId, "default");
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
const streamKey = `stream:${runId}:${streamId}`;
|
||||
const entries = await redis.xrange(streamKey, "-", "+");
|
||||
expect(entries.length).toBe(3);
|
||||
|
||||
const lastChunkIndex = await streams.getLastChunkIndex(runId, streamId, "default");
|
||||
expect(lastChunkIndex).toBe(2);
|
||||
|
||||
await redis.del(streamKey);
|
||||
await redis.quit();
|
||||
}
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user