chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
# Replication
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@internal/replication",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@internal/redis": "workspace:*",
|
||||
"@internal/tracing": "workspace:*",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"pg": "8.15.6",
|
||||
"redlock": "5.0.0-beta.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internal/testcontainers": "workspace:*",
|
||||
"rimraf": "6.0.1",
|
||||
"@types/pg": "8.11.14"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.build.json",
|
||||
"build": "pnpm run clean && tsc -p tsconfig.build.json",
|
||||
"dev": "tsc --watch -p tsconfig.build.json",
|
||||
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
|
||||
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
import { postgresAndRedisTest } from "@internal/testcontainers";
|
||||
import { LogicalReplicationClient } from "./client.js";
|
||||
import { setTimeout } from "timers/promises";
|
||||
|
||||
describe("Replication Client", () => {
|
||||
postgresAndRedisTest(
|
||||
"should be able to subscribe to changes on a table",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const client = new LogicalReplicationClient({
|
||||
name: "test",
|
||||
slotName: "test_slot",
|
||||
publicationName: "test_publication",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: {
|
||||
connectionString: postgresContainer.getConnectionUri(),
|
||||
},
|
||||
});
|
||||
|
||||
const logs: Array<{
|
||||
lsn: string;
|
||||
log: unknown;
|
||||
}> = [];
|
||||
|
||||
client.events.on("data", (data) => {
|
||||
console.log(data);
|
||||
logs.push(data);
|
||||
});
|
||||
|
||||
client.events.on("error", (error) => {
|
||||
console.error(error);
|
||||
});
|
||||
|
||||
await client.subscribe();
|
||||
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "test",
|
||||
slug: "test",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "test",
|
||||
slug: "test",
|
||||
organizationId: organization.id,
|
||||
externalRef: "test",
|
||||
},
|
||||
});
|
||||
|
||||
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "test",
|
||||
type: "DEVELOPMENT",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "test",
|
||||
pkApiKey: "test",
|
||||
shortcode: "test",
|
||||
},
|
||||
});
|
||||
|
||||
// Now we insert a row into the table
|
||||
await prisma.taskRun.create({
|
||||
data: {
|
||||
friendlyId: "run_1234",
|
||||
taskIdentifier: "my-task",
|
||||
payload: JSON.stringify({ foo: "bar" }),
|
||||
traceId: "1234",
|
||||
spanId: "1234",
|
||||
queue: "test",
|
||||
runtimeEnvironmentId: runtimeEnvironment.id,
|
||||
projectId: project.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Wait for a bit of time
|
||||
await setTimeout(50);
|
||||
|
||||
// Now we should see the row in the logs
|
||||
expect(logs.length).toBeGreaterThan(0);
|
||||
|
||||
await client.stop();
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"should be able to teardown",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const client = new LogicalReplicationClient({
|
||||
name: "test",
|
||||
slotName: "test_slot",
|
||||
publicationName: "test_publication",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: {
|
||||
connectionString: postgresContainer.getConnectionUri(),
|
||||
},
|
||||
});
|
||||
|
||||
const logs: Array<{
|
||||
lsn: string;
|
||||
log: unknown;
|
||||
}> = [];
|
||||
|
||||
client.events.on("data", (data) => {
|
||||
console.log(data);
|
||||
logs.push(data);
|
||||
});
|
||||
|
||||
client.events.on("error", (error) => {
|
||||
console.error(error);
|
||||
});
|
||||
|
||||
await client.subscribe();
|
||||
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "test",
|
||||
slug: "test",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "test",
|
||||
slug: "test",
|
||||
organizationId: organization.id,
|
||||
externalRef: "test",
|
||||
},
|
||||
});
|
||||
|
||||
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "test",
|
||||
type: "DEVELOPMENT",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "test",
|
||||
pkApiKey: "test",
|
||||
shortcode: "test",
|
||||
},
|
||||
});
|
||||
|
||||
// Now we insert a row into the table
|
||||
await prisma.taskRun.create({
|
||||
data: {
|
||||
friendlyId: "run_1234",
|
||||
taskIdentifier: "my-task",
|
||||
payload: JSON.stringify({ foo: "bar" }),
|
||||
traceId: "1234",
|
||||
spanId: "1234",
|
||||
queue: "test",
|
||||
runtimeEnvironmentId: runtimeEnvironment.id,
|
||||
projectId: project.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Wait for a bit of time
|
||||
await setTimeout(50);
|
||||
|
||||
// Now we should see the row in the logs
|
||||
expect(logs.length).toBeGreaterThan(0);
|
||||
|
||||
const slotDropped = await client.teardown();
|
||||
|
||||
expect(slotDropped).toBe(true);
|
||||
|
||||
// Now the replication slot should be gone
|
||||
const slotExists = await prisma.$queryRaw<
|
||||
{ exists: boolean }[]
|
||||
>`SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_slot');`;
|
||||
|
||||
console.log(slotExists);
|
||||
|
||||
expect(slotExists[0].exists).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"two clients on the same slot must not both lead (rolling-deploy handoff)",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const shared = {
|
||||
slotName: "handoff_slot",
|
||||
publicationName: "handoff_publication",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: { connectionString: postgresContainer.getConnectionUri() },
|
||||
};
|
||||
|
||||
// Leader on the shared slot.
|
||||
const a = new LogicalReplicationClient({ ...shared, name: "runs-replication" });
|
||||
const aElections: boolean[] = [];
|
||||
a.events.on("leaderElection", (won) => aElections.push(won));
|
||||
a.events.on("error", () => {});
|
||||
await a.subscribe();
|
||||
// Let A's walsender actually attach to the slot before B races it.
|
||||
await setTimeout(1000);
|
||||
|
||||
// Second client, SAME slot, DIFFERENT name — the rolling-deploy shape that
|
||||
// regressed (name changed "runs-replication" -> "runs-replication:legacy").
|
||||
const b = new LogicalReplicationClient({
|
||||
...shared,
|
||||
name: "runs-replication:legacy",
|
||||
leaderLockTimeoutMs: 1000,
|
||||
leaderLockAcquireAdditionalTimeMs: 250,
|
||||
leaderLockRetryIntervalMs: 200,
|
||||
});
|
||||
const bElections: boolean[] = [];
|
||||
const bErrors: Array<unknown> = [];
|
||||
b.events.on("leaderElection", (won) => bElections.push(won));
|
||||
b.events.on("error", (error) => bErrors.push(error));
|
||||
await b.subscribe();
|
||||
await setTimeout(500);
|
||||
|
||||
expect(aElections).toContain(true);
|
||||
// B must not also win leadership on the same slot, nor race START_REPLICATION
|
||||
// into a "slot is active" error. With a name-keyed lock it did both.
|
||||
expect(bElections).not.toContain(true);
|
||||
expect(bElections).toContain(false);
|
||||
expect(
|
||||
bErrors
|
||||
.map((e) => String((e as Error)?.message ?? e))
|
||||
.some((m) => /replication slot .* is active|already active/i.test(m))
|
||||
).toBe(false);
|
||||
|
||||
await a.stop();
|
||||
await b.stop();
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"resubscribeOnFailure self-heals once the leader releases the slot",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const shared = {
|
||||
slotName: "resub_slot",
|
||||
publicationName: "resub_pub",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: { connectionString: postgresContainer.getConnectionUri() },
|
||||
};
|
||||
|
||||
// Leader holds the slot.
|
||||
const a = new LogicalReplicationClient({ ...shared, name: "leader-a" });
|
||||
a.events.on("error", () => {});
|
||||
await a.subscribe();
|
||||
await setTimeout(1000);
|
||||
|
||||
// Contender with resubscribe on: loses the election while A holds the slot,
|
||||
// then must self-heal (win) once A releases it — the rolling-deploy handoff.
|
||||
const b = new LogicalReplicationClient({
|
||||
...shared,
|
||||
name: "contender-b",
|
||||
resubscribeOnFailure: true,
|
||||
resubscribeMinDelayMs: 200,
|
||||
resubscribeMaxDelayMs: 400,
|
||||
leaderLockTimeoutMs: 500,
|
||||
leaderLockAcquireAdditionalTimeMs: 100,
|
||||
leaderLockRetryIntervalMs: 100,
|
||||
});
|
||||
const bElections: boolean[] = [];
|
||||
b.events.on("leaderElection", (won) => bElections.push(won));
|
||||
b.events.on("error", () => {});
|
||||
await b.subscribe();
|
||||
await setTimeout(1500);
|
||||
|
||||
// Still contending, not leader, while A holds the slot.
|
||||
expect(bElections).toContain(false);
|
||||
expect(bElections).not.toContain(true);
|
||||
|
||||
// Release the leader — a scheduled resubscribe should now win.
|
||||
await a.shutdown();
|
||||
|
||||
let becameLeader = false;
|
||||
for (let i = 0; i < 40; i++) {
|
||||
if (bElections.includes(true)) {
|
||||
becameLeader = true;
|
||||
break;
|
||||
}
|
||||
await setTimeout(250);
|
||||
}
|
||||
expect(becameLeader).toBe(true);
|
||||
|
||||
await b.shutdown();
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"a failing START_REPLICATION retry loop must not leak connections or locks",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const shared = {
|
||||
slotName: "leak_slot",
|
||||
publicationName: "leak_pub",
|
||||
table: "TaskRun",
|
||||
pgConfig: { connectionString: postgresContainer.getConnectionUri() },
|
||||
};
|
||||
|
||||
const a = new LogicalReplicationClient({ ...shared, redisOptions, name: "leak-leader" });
|
||||
a.events.on("error", () => {});
|
||||
await a.subscribe();
|
||||
await setTimeout(1000);
|
||||
|
||||
// B elects on a separate lock namespace so every attempt reaches
|
||||
// START_REPLICATION and dies there ("slot is active") — the stuck-slot shape.
|
||||
const b = new LogicalReplicationClient({
|
||||
...shared,
|
||||
redisOptions: { ...redisOptions, keyPrefix: `${redisOptions.keyPrefix ?? ""}other:` },
|
||||
name: "leak-contender",
|
||||
resubscribeOnFailure: true,
|
||||
resubscribeMinDelayMs: 200,
|
||||
resubscribeMaxDelayMs: 400,
|
||||
leaderLockTimeoutMs: 1000,
|
||||
leaderLockAcquireAdditionalTimeMs: 300,
|
||||
leaderLockRetryIntervalMs: 100,
|
||||
});
|
||||
const bErrors: Array<unknown> = [];
|
||||
b.events.on("error", (error) => bErrors.push(error));
|
||||
await b.subscribe();
|
||||
|
||||
for (let i = 0; i < 80 && bErrors.length < 3; i++) {
|
||||
await setTimeout(250);
|
||||
}
|
||||
expect(bErrors.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Every failed attempt must end its pg client: at most the one in-flight
|
||||
// attempt's backend may exist, never an accrual across cycles.
|
||||
const backends = await prisma.$queryRaw<{ count: bigint }[]>`
|
||||
SELECT count(*) AS count FROM pg_stat_activity WHERE application_name = 'leak-contender'
|
||||
`;
|
||||
expect(Number(backends[0].count)).toBeLessThanOrEqual(1);
|
||||
|
||||
const active = await prisma.$queryRaw<{ count: bigint }[]>`
|
||||
SELECT count(*) AS count FROM pg_replication_slots WHERE slot_name = 'leak_slot' AND active
|
||||
`;
|
||||
expect(Number(active[0].count)).toBe(1);
|
||||
|
||||
await b.shutdown();
|
||||
await a.shutdown();
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"shutdown during an in-flight subscribe must not leave a zombie leader",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const shared = {
|
||||
slotName: "zombie_slot",
|
||||
publicationName: "zombie_pub",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: { connectionString: postgresContainer.getConnectionUri() },
|
||||
};
|
||||
|
||||
const a = new LogicalReplicationClient({ ...shared, name: "zombie-leader" });
|
||||
a.events.on("error", () => {});
|
||||
await a.subscribe();
|
||||
await setTimeout(1000);
|
||||
|
||||
// B's election spins against A's held lock; shut it down mid-subscribe.
|
||||
const b = new LogicalReplicationClient({
|
||||
...shared,
|
||||
name: "zombie-contender",
|
||||
resubscribeOnFailure: true,
|
||||
leaderLockTimeoutMs: 5000,
|
||||
leaderLockAcquireAdditionalTimeMs: 5000,
|
||||
leaderLockRetryIntervalMs: 100,
|
||||
});
|
||||
const bElections: boolean[] = [];
|
||||
b.events.on("leaderElection", (won) => bElections.push(won));
|
||||
b.events.on("error", () => {});
|
||||
|
||||
const inflight = b.subscribe();
|
||||
await setTimeout(300);
|
||||
await b.shutdown();
|
||||
|
||||
// Release the real leader; a zombie B would now win the lock and the slot.
|
||||
await a.shutdown();
|
||||
await inflight.catch(() => {});
|
||||
await setTimeout(1500);
|
||||
|
||||
const zombieWon = bElections.includes(true);
|
||||
const active = await prisma.$queryRaw<{ count: bigint }[]>`
|
||||
SELECT count(*) AS count FROM pg_replication_slots WHERE slot_name = 'zombie_slot' AND active
|
||||
`;
|
||||
const backends = await prisma.$queryRaw<{ count: bigint }[]>`
|
||||
SELECT count(*) AS count FROM pg_stat_activity WHERE application_name = 'zombie-contender'
|
||||
`;
|
||||
// Reap a zombie (if any) so the test exits cleanly, then assert.
|
||||
await b.shutdown();
|
||||
|
||||
expect(zombieWon).toBe(false);
|
||||
expect(Number(active[0].count)).toBe(0);
|
||||
expect(Number(backends[0].count)).toBe(0);
|
||||
}
|
||||
);
|
||||
|
||||
postgresAndRedisTest(
|
||||
"subscribe after shutdown re-arms resubscribeOnFailure",
|
||||
async ({ postgresContainer, prisma, redisOptions }) => {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
|
||||
|
||||
const shared = {
|
||||
slotName: "rearm_slot",
|
||||
publicationName: "rearm_pub",
|
||||
redisOptions,
|
||||
table: "TaskRun",
|
||||
pgConfig: { connectionString: postgresContainer.getConnectionUri() },
|
||||
};
|
||||
|
||||
const b = new LogicalReplicationClient({
|
||||
...shared,
|
||||
name: "rearm-client",
|
||||
resubscribeOnFailure: true,
|
||||
resubscribeMinDelayMs: 200,
|
||||
resubscribeMaxDelayMs: 400,
|
||||
leaderLockTimeoutMs: 500,
|
||||
leaderLockAcquireAdditionalTimeMs: 100,
|
||||
leaderLockRetryIntervalMs: 100,
|
||||
});
|
||||
const bElections: boolean[] = [];
|
||||
b.events.on("leaderElection", (won) => bElections.push(won));
|
||||
b.events.on("error", () => {});
|
||||
|
||||
// Admin stop -> start: shutdown latches the intentional stop...
|
||||
await b.subscribe();
|
||||
await setTimeout(500);
|
||||
await b.shutdown();
|
||||
|
||||
const a = new LogicalReplicationClient({ ...shared, name: "rearm-leader" });
|
||||
a.events.on("error", () => {});
|
||||
await a.subscribe();
|
||||
await setTimeout(1000);
|
||||
|
||||
// ...then an explicit re-subscribe loses the election and must self-heal
|
||||
// once the leader goes away (self-heal re-armed by the subscribe).
|
||||
bElections.length = 0;
|
||||
await b.subscribe();
|
||||
expect(bElections).toContain(false);
|
||||
expect(bElections).not.toContain(true);
|
||||
|
||||
await a.shutdown();
|
||||
|
||||
let becameLeader = false;
|
||||
for (let i = 0; i < 40; i++) {
|
||||
if (bElections.includes(true)) {
|
||||
becameLeader = true;
|
||||
break;
|
||||
}
|
||||
await setTimeout(250);
|
||||
}
|
||||
expect(becameLeader).toBe(true);
|
||||
|
||||
await b.shutdown();
|
||||
}
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
export class LogicalReplicationClientError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./client.js";
|
||||
export * from "./errors.js";
|
||||
export type * from "./pgoutput.js";
|
||||
@@ -0,0 +1,546 @@
|
||||
// NOTE: This file requires ES2020 or higher for BigInt literals (used in BinaryReader.readTime)
|
||||
import { types } from "pg";
|
||||
|
||||
export interface PgoutputOptions {
|
||||
protoVersion: 1 | 2;
|
||||
publicationNames: string[];
|
||||
messages?: boolean;
|
||||
}
|
||||
|
||||
export type PgoutputMessage =
|
||||
| MessageBegin
|
||||
| MessageCommit
|
||||
| MessageDelete
|
||||
| MessageInsert
|
||||
| MessageMessage
|
||||
| MessageOrigin
|
||||
| MessageRelation
|
||||
| MessageTruncate
|
||||
| MessageType
|
||||
| MessageUpdate;
|
||||
|
||||
export type PgoutputMessageArray =
|
||||
| MessageBegin
|
||||
| MessageCommit
|
||||
| MessageDeleteArray
|
||||
| MessageInsertArray
|
||||
| MessageMessage
|
||||
| MessageOrigin
|
||||
| MessageRelation
|
||||
| MessageTruncate
|
||||
| MessageType
|
||||
| MessageUpdateArray;
|
||||
|
||||
export interface MessageBegin {
|
||||
tag: "begin";
|
||||
commitLsn: string | null;
|
||||
commitTime: bigint;
|
||||
xid: number;
|
||||
}
|
||||
export interface MessageCommit {
|
||||
tag: "commit";
|
||||
flags: number;
|
||||
commitLsn: string | null;
|
||||
commitEndLsn: string | null;
|
||||
commitTime: bigint;
|
||||
}
|
||||
export interface MessageDelete {
|
||||
tag: "delete";
|
||||
relation: MessageRelation;
|
||||
key: Record<string, any> | null;
|
||||
old: Record<string, any> | null;
|
||||
}
|
||||
export interface MessageInsert {
|
||||
tag: "insert";
|
||||
relation: MessageRelation;
|
||||
new: Record<string, any>;
|
||||
}
|
||||
export interface MessageMessage {
|
||||
tag: "message";
|
||||
flags: number;
|
||||
transactional: boolean;
|
||||
messageLsn: string | null;
|
||||
prefix: string;
|
||||
content: Uint8Array;
|
||||
}
|
||||
export interface MessageOrigin {
|
||||
tag: "origin";
|
||||
originLsn: string | null;
|
||||
originName: string;
|
||||
}
|
||||
export interface MessageRelation {
|
||||
tag: "relation";
|
||||
relationOid: number;
|
||||
schema: string;
|
||||
name: string;
|
||||
replicaIdentity: "default" | "nothing" | "full" | "index";
|
||||
columns: RelationColumn[];
|
||||
keyColumns: string[];
|
||||
}
|
||||
export interface RelationColumn {
|
||||
name: string;
|
||||
flags: number;
|
||||
typeOid: number;
|
||||
typeMod: number;
|
||||
typeSchema: string | null;
|
||||
typeName: string | null;
|
||||
parser: (raw: any) => any;
|
||||
}
|
||||
export interface MessageTruncate {
|
||||
tag: "truncate";
|
||||
cascade: boolean;
|
||||
restartIdentity: boolean;
|
||||
relations: MessageRelation[];
|
||||
}
|
||||
export interface MessageType {
|
||||
tag: "type";
|
||||
typeOid: number;
|
||||
typeSchema: string;
|
||||
typeName: string;
|
||||
}
|
||||
export interface MessageUpdate {
|
||||
tag: "update";
|
||||
relation: MessageRelation;
|
||||
key: Record<string, any> | null;
|
||||
old: Record<string, any> | null;
|
||||
new: Record<string, any>;
|
||||
}
|
||||
|
||||
// Array variants for zero-copy performance
|
||||
export interface MessageInsertArray {
|
||||
tag: "insert";
|
||||
relation: MessageRelation;
|
||||
new: any[];
|
||||
}
|
||||
export interface MessageUpdateArray {
|
||||
tag: "update";
|
||||
relation: MessageRelation;
|
||||
key: any[] | null;
|
||||
old: any[] | null;
|
||||
new: any[];
|
||||
}
|
||||
export interface MessageDeleteArray {
|
||||
tag: "delete";
|
||||
relation: MessageRelation;
|
||||
key: any[] | null;
|
||||
old: any[] | null;
|
||||
}
|
||||
|
||||
class BinaryReader {
|
||||
private offset = 0;
|
||||
constructor(private buf: Buffer) {}
|
||||
readUint8(): number {
|
||||
return this.buf.readUInt8(this.offset++);
|
||||
}
|
||||
readInt16(): number {
|
||||
const v = this.buf.readInt16BE(this.offset);
|
||||
this.offset += 2;
|
||||
return v;
|
||||
}
|
||||
readInt32(): number {
|
||||
const v = this.buf.readInt32BE(this.offset);
|
||||
this.offset += 4;
|
||||
return v;
|
||||
}
|
||||
readString(): string {
|
||||
let end = this.buf.indexOf(0, this.offset);
|
||||
if (end === -1) throw new Error("Null-terminated string not found");
|
||||
const str = this.buf.toString("utf8", this.offset, end);
|
||||
this.offset = end + 1;
|
||||
return str;
|
||||
}
|
||||
read(len: number): Buffer {
|
||||
const b = this.buf.subarray(this.offset, this.offset + len);
|
||||
this.offset += len;
|
||||
return b;
|
||||
}
|
||||
decodeText(buf: Buffer): string {
|
||||
return buf.toString("utf8");
|
||||
}
|
||||
array<T>(n: number, fn: () => T): T[] {
|
||||
return Array.from({ length: n }, fn);
|
||||
}
|
||||
|
||||
readLsn(): string | null {
|
||||
const upper = this.readUint32();
|
||||
const lower = this.readUint32();
|
||||
if (upper === 0 && lower === 0) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
upper.toString(16).padStart(8, "0").toUpperCase() +
|
||||
"/" +
|
||||
lower.toString(16).padStart(8, "0").toUpperCase()
|
||||
);
|
||||
}
|
||||
|
||||
readUint32(): number {
|
||||
// >>> 0 ensures unsigned
|
||||
return this.readInt32() >>> 0;
|
||||
}
|
||||
|
||||
readUint64(): bigint {
|
||||
// Combine two unsigned 32-bit ints into a 64-bit bigint
|
||||
return (BigInt(this.readUint32()) << 32n) | BigInt(this.readUint32());
|
||||
}
|
||||
|
||||
readTime(): bigint {
|
||||
// (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY == 946684800000000
|
||||
const microsSinceUnixEpoch = this.readUint64() + 946684800000000n;
|
||||
return microsSinceUnixEpoch;
|
||||
}
|
||||
}
|
||||
|
||||
export class PgoutputParser {
|
||||
private _typeCache = new Map<number, { typeSchema: string; typeName: string }>();
|
||||
private _relationCache = new Map<number, MessageRelation>();
|
||||
|
||||
public parse(buf: Buffer): PgoutputMessage {
|
||||
const reader = new BinaryReader(buf);
|
||||
const tag = reader.readUint8();
|
||||
switch (tag) {
|
||||
case 0x42:
|
||||
return this.msgBegin(reader);
|
||||
case 0x4f:
|
||||
return this.msgOrigin(reader);
|
||||
case 0x59:
|
||||
return this.msgType(reader);
|
||||
case 0x52:
|
||||
return this.msgRelation(reader);
|
||||
case 0x49:
|
||||
return this.msgInsert(reader);
|
||||
case 0x55:
|
||||
return this.msgUpdate(reader);
|
||||
case 0x44:
|
||||
return this.msgDelete(reader);
|
||||
case 0x54:
|
||||
return this.msgTruncate(reader);
|
||||
case 0x4d:
|
||||
return this.msgMessage(reader);
|
||||
case 0x43:
|
||||
return this.msgCommit(reader);
|
||||
default:
|
||||
throw Error("unknown pgoutput message");
|
||||
}
|
||||
}
|
||||
|
||||
public parseArray(buf: Buffer): PgoutputMessageArray {
|
||||
const reader = new BinaryReader(buf);
|
||||
const tag = reader.readUint8();
|
||||
switch (tag) {
|
||||
case 0x42:
|
||||
return this.msgBegin(reader);
|
||||
case 0x4f:
|
||||
return this.msgOrigin(reader);
|
||||
case 0x59:
|
||||
return this.msgType(reader);
|
||||
case 0x52:
|
||||
return this.msgRelation(reader);
|
||||
case 0x49:
|
||||
return this.msgInsertArray(reader);
|
||||
case 0x55:
|
||||
return this.msgUpdateArray(reader);
|
||||
case 0x44:
|
||||
return this.msgDeleteArray(reader);
|
||||
case 0x54:
|
||||
return this.msgTruncate(reader);
|
||||
case 0x4d:
|
||||
return this.msgMessage(reader);
|
||||
case 0x43:
|
||||
return this.msgCommit(reader);
|
||||
default:
|
||||
throw Error("unknown pgoutput message");
|
||||
}
|
||||
}
|
||||
|
||||
private msgBegin(reader: BinaryReader): MessageBegin {
|
||||
return {
|
||||
tag: "begin",
|
||||
commitLsn: reader.readLsn(),
|
||||
commitTime: reader.readTime(),
|
||||
xid: reader.readInt32(),
|
||||
};
|
||||
}
|
||||
private msgOrigin(reader: BinaryReader): MessageOrigin {
|
||||
return {
|
||||
tag: "origin",
|
||||
originLsn: reader.readLsn(),
|
||||
originName: reader.readString(),
|
||||
};
|
||||
}
|
||||
private msgType(reader: BinaryReader): MessageType {
|
||||
const typeOid = reader.readInt32();
|
||||
const typeSchema = reader.readString();
|
||||
const typeName = reader.readString();
|
||||
this._typeCache.set(typeOid, { typeSchema, typeName });
|
||||
return { tag: "type", typeOid, typeSchema, typeName };
|
||||
}
|
||||
private msgRelation(reader: BinaryReader): MessageRelation {
|
||||
const relationOid = reader.readInt32();
|
||||
const schema = reader.readString();
|
||||
const name = reader.readString();
|
||||
const replicaIdentity = this.readRelationReplicaIdentity(reader);
|
||||
const columns = reader.array(reader.readInt16(), () => this.readRelationColumn(reader));
|
||||
const keyColumns = columns.filter((it) => it.flags & 0b1).map((it) => it.name);
|
||||
const msg: MessageRelation = {
|
||||
tag: "relation",
|
||||
relationOid,
|
||||
schema,
|
||||
name,
|
||||
replicaIdentity,
|
||||
columns,
|
||||
keyColumns,
|
||||
};
|
||||
this._relationCache.set(relationOid, msg);
|
||||
return msg;
|
||||
}
|
||||
private readRelationReplicaIdentity(reader: BinaryReader) {
|
||||
const ident = reader.readUint8();
|
||||
switch (ident) {
|
||||
case 0x64:
|
||||
return "default";
|
||||
case 0x6e:
|
||||
return "nothing";
|
||||
case 0x66:
|
||||
return "full";
|
||||
case 0x69:
|
||||
return "index";
|
||||
default:
|
||||
throw Error(`unknown replica identity ${String.fromCharCode(ident)}`);
|
||||
}
|
||||
}
|
||||
private readRelationColumn(reader: BinaryReader): RelationColumn {
|
||||
const flags = reader.readUint8();
|
||||
const name = reader.readString();
|
||||
const typeOid = reader.readInt32();
|
||||
const typeMod = reader.readInt32();
|
||||
return {
|
||||
flags,
|
||||
name,
|
||||
typeOid,
|
||||
typeMod,
|
||||
typeSchema: null,
|
||||
typeName: null,
|
||||
...this._typeCache.get(typeOid),
|
||||
parser: types.getTypeParser(typeOid),
|
||||
};
|
||||
}
|
||||
private msgInsert(reader: BinaryReader): MessageInsert {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
reader.readUint8(); // consume the 'N' key
|
||||
return {
|
||||
tag: "insert",
|
||||
relation,
|
||||
new: this.readTuple(reader, relation),
|
||||
};
|
||||
}
|
||||
private msgUpdate(reader: BinaryReader): MessageUpdate {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
let key: Record<string, any> | null = null;
|
||||
let old: Record<string, any> | null = null;
|
||||
let new_: Record<string, any> | null = null;
|
||||
const subMsgKey = reader.readUint8();
|
||||
if (subMsgKey === 0x4b) {
|
||||
key = this.readKeyTuple(reader, relation);
|
||||
reader.readUint8();
|
||||
new_ = this.readTuple(reader, relation);
|
||||
} else if (subMsgKey === 0x4f) {
|
||||
old = this.readTuple(reader, relation);
|
||||
reader.readUint8();
|
||||
new_ = this.readTuple(reader, relation, old);
|
||||
} else if (subMsgKey === 0x4e) {
|
||||
new_ = this.readTuple(reader, relation);
|
||||
} else {
|
||||
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
|
||||
}
|
||||
return { tag: "update", relation, key, old, new: new_ };
|
||||
}
|
||||
private msgDelete(reader: BinaryReader): MessageDelete {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
let key: Record<string, any> | null = null;
|
||||
let old: Record<string, any> | null = null;
|
||||
const subMsgKey = reader.readUint8();
|
||||
if (subMsgKey === 0x4b) {
|
||||
key = this.readKeyTuple(reader, relation);
|
||||
} else if (subMsgKey === 0x4f) {
|
||||
old = this.readTuple(reader, relation);
|
||||
} else {
|
||||
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
|
||||
}
|
||||
return { tag: "delete", relation, key, old };
|
||||
}
|
||||
|
||||
// Array variants - skip object creation for performance
|
||||
private msgInsertArray(reader: BinaryReader): MessageInsertArray {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
reader.readUint8(); // consume the 'N' key
|
||||
return {
|
||||
tag: "insert",
|
||||
relation,
|
||||
new: this.readTupleAsArray(reader, relation),
|
||||
};
|
||||
}
|
||||
private msgUpdateArray(reader: BinaryReader): MessageUpdateArray {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
let key: any[] | null = null;
|
||||
let old: any[] | null = null;
|
||||
let new_: any[] | null = null;
|
||||
const subMsgKey = reader.readUint8();
|
||||
if (subMsgKey === 0x4b) {
|
||||
key = this.readTupleAsArray(reader, relation);
|
||||
reader.readUint8();
|
||||
new_ = this.readTupleAsArray(reader, relation);
|
||||
} else if (subMsgKey === 0x4f) {
|
||||
old = this.readTupleAsArray(reader, relation);
|
||||
reader.readUint8();
|
||||
new_ = this.readTupleAsArray(reader, relation, old);
|
||||
} else if (subMsgKey === 0x4e) {
|
||||
new_ = this.readTupleAsArray(reader, relation);
|
||||
} else {
|
||||
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
|
||||
}
|
||||
return { tag: "update", relation, key, old, new: new_ };
|
||||
}
|
||||
private msgDeleteArray(reader: BinaryReader): MessageDeleteArray {
|
||||
const relation = this._relationCache.get(reader.readInt32());
|
||||
if (!relation) throw Error("missing relation");
|
||||
let key: any[] | null = null;
|
||||
let old: any[] | null = null;
|
||||
const subMsgKey = reader.readUint8();
|
||||
if (subMsgKey === 0x4b) {
|
||||
key = this.readTupleAsArray(reader, relation);
|
||||
} else if (subMsgKey === 0x4f) {
|
||||
old = this.readTupleAsArray(reader, relation);
|
||||
} else {
|
||||
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
|
||||
}
|
||||
return { tag: "delete", relation, key, old };
|
||||
}
|
||||
private readKeyTuple(reader: BinaryReader, relation: MessageRelation): Record<string, any> {
|
||||
const tuple = this.readTuple(reader, relation);
|
||||
const key = Object.create(null);
|
||||
for (const k of relation.keyColumns) {
|
||||
key[k] = tuple[k] === null ? undefined : tuple[k];
|
||||
}
|
||||
return key;
|
||||
}
|
||||
private readTuple(
|
||||
reader: BinaryReader,
|
||||
{ columns }: MessageRelation,
|
||||
unchangedToastFallback?: Record<string, any> | null
|
||||
): Record<string, any> {
|
||||
const nfields = reader.readInt16();
|
||||
const tuple = Object.create(null);
|
||||
for (let i = 0; i < nfields; i++) {
|
||||
const { name, parser } = columns[i];
|
||||
const kind = reader.readUint8();
|
||||
switch (kind) {
|
||||
case 0x62: // 'b' binary
|
||||
const bsize = reader.readInt32();
|
||||
const bval = reader.read(bsize);
|
||||
tuple[name] = bval;
|
||||
break;
|
||||
case 0x74: // 't' text
|
||||
const valsize = reader.readInt32();
|
||||
const valbuf = reader.read(valsize);
|
||||
const valtext = reader.decodeText(valbuf);
|
||||
tuple[name] = parser(valtext);
|
||||
break;
|
||||
case 0x6e: // 'n' null
|
||||
tuple[name] = null;
|
||||
break;
|
||||
case 0x75: // 'u' unchanged toast datum
|
||||
tuple[name] = unchangedToastFallback?.[name];
|
||||
break;
|
||||
default:
|
||||
throw Error(`unknown attribute kind ${String.fromCharCode(kind)}`);
|
||||
}
|
||||
}
|
||||
return tuple;
|
||||
}
|
||||
|
||||
private readTupleAsArray(
|
||||
reader: BinaryReader,
|
||||
{ columns }: MessageRelation,
|
||||
unchangedToastFallback?: any[] | null
|
||||
): any[] {
|
||||
const nfields = reader.readInt16();
|
||||
const tuple = new Array(nfields);
|
||||
for (let i = 0; i < nfields; i++) {
|
||||
const { parser } = columns[i];
|
||||
const kind = reader.readUint8();
|
||||
switch (kind) {
|
||||
case 0x62: // 'b' binary
|
||||
const bsize = reader.readInt32();
|
||||
tuple[i] = reader.read(bsize);
|
||||
break;
|
||||
case 0x74: // 't' text
|
||||
const valsize = reader.readInt32();
|
||||
const valbuf = reader.read(valsize);
|
||||
const valtext = reader.decodeText(valbuf);
|
||||
tuple[i] = parser(valtext);
|
||||
break;
|
||||
case 0x6e: // 'n' null
|
||||
tuple[i] = null;
|
||||
break;
|
||||
case 0x75: // 'u' unchanged toast datum
|
||||
tuple[i] = unchangedToastFallback?.[i];
|
||||
break;
|
||||
default:
|
||||
throw Error(`unknown attribute kind ${String.fromCharCode(kind)}`);
|
||||
}
|
||||
}
|
||||
return tuple;
|
||||
}
|
||||
private msgTruncate(reader: BinaryReader): MessageTruncate {
|
||||
const nrels = reader.readInt32();
|
||||
const flags = reader.readUint8();
|
||||
return {
|
||||
tag: "truncate",
|
||||
cascade: Boolean(flags & 0b1),
|
||||
restartIdentity: Boolean(flags & 0b10),
|
||||
relations: reader.array(
|
||||
nrels,
|
||||
() => this._relationCache.get(reader.readInt32()) as MessageRelation
|
||||
),
|
||||
};
|
||||
}
|
||||
private msgMessage(reader: BinaryReader): MessageMessage {
|
||||
const flags = reader.readUint8();
|
||||
return {
|
||||
tag: "message",
|
||||
flags,
|
||||
transactional: Boolean(flags & 0b1),
|
||||
messageLsn: reader.readLsn(),
|
||||
prefix: reader.readString(),
|
||||
content: reader.read(reader.readInt32()),
|
||||
};
|
||||
}
|
||||
private msgCommit(reader: BinaryReader): MessageCommit {
|
||||
return {
|
||||
tag: "commit",
|
||||
flags: reader.readUint8(),
|
||||
commitLsn: reader.readLsn(),
|
||||
commitEndLsn: reader.readLsn(),
|
||||
commitTime: reader.readTime(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function getPgoutputStartReplicationSQL(
|
||||
slotName: string,
|
||||
lastLsn: string,
|
||||
options: PgoutputOptions
|
||||
): string {
|
||||
const opts = [
|
||||
`proto_version '${options.protoVersion}'`,
|
||||
`publication_names '${options.publicationNames.join(",")}'`,
|
||||
`messages '${options.messages ?? false}'`,
|
||||
];
|
||||
return `START_REPLICATION SLOT "${slotName}" LOGICAL ${lastLsn} (${opts.join(", ")});`;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"outDir": "dist",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"declaration": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "Node16",
|
||||
"module": "Node16",
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.test.ts", "vitest.config.ts"],
|
||||
"references": [{ "path": "./tsconfig.src.json" }],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"types": ["vitest/globals"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
sequence: { sequencer: DurationShardingSequencer },
|
||||
include: ["**/*.test.ts"],
|
||||
globals: true,
|
||||
isolate: true,
|
||||
fileParallelism: false,
|
||||
testTimeout: 60_000,
|
||||
coverage: {
|
||||
provider: "v8",
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user