chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,695 @@
|
||||
import { type ChangeRecord } from "./runChangeNotifier.server";
|
||||
import { type RealtimeRunRow, serializeRunRow } from "./electricStreamProtocol.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
/**
|
||||
* EnvChangeRouter — per-instance routing layer that fans one env's change stream out to the feeds it
|
||||
* matches. Owns one subscription per env (over the RunChangeNotifier) plus an inverted index of held
|
||||
* feeds, then per batch: routes via the index, batch-hydrates matched runs once per column set,
|
||||
* serializes each row's wire value once, and resolves each matched feed's pending wait. Stateless across reconnects.
|
||||
*/
|
||||
|
||||
export type WakeReason = "notify" | "timeout" | "abort";
|
||||
|
||||
/** A feed's membership predicate over the env stream. */
|
||||
export type FeedFilter =
|
||||
| { kind: "run"; runId: string }
|
||||
| { kind: "tag"; tags: string[]; createdAtFloorMs?: number }
|
||||
| { kind: "batch"; batchId: string };
|
||||
|
||||
/** A matched run handed to a feed: the hydrated row (for the feed's working-set diff) and
|
||||
* its wire `value` serialized once for this feed's column set (shared across feeds). */
|
||||
export type MatchedRow = { row: RealtimeRunRow; value: Record<string, string | null> };
|
||||
|
||||
export type WaitResult = { reason: WakeReason; rows: MatchedRow[] };
|
||||
|
||||
/** Minimal deps so the router is unit-testable without Redis/Postgres. */
|
||||
export interface EnvChangeSource {
|
||||
subscribeToEnv(environmentId: string, onBatch: (records: ChangeRecord[]) => void): () => void;
|
||||
}
|
||||
export interface RowHydrator {
|
||||
hydrateByIds(
|
||||
environmentId: string,
|
||||
ids: string[],
|
||||
skipColumns: string[]
|
||||
): Promise<RealtimeRunRow[]>;
|
||||
}
|
||||
|
||||
export type EnvChangeRouterOptions = {
|
||||
source: EnvChangeSource;
|
||||
hydrator: RowHydrator;
|
||||
/** Observability: a hydrate-by-id batch ran (count = runs hydrated this tick). */
|
||||
onHydrate?: (runCount: number) => void;
|
||||
/** How far back (ms) a newly-armed feed replays buffered records. 0 disables replay. */
|
||||
replayWindowMs?: number;
|
||||
/** Cap on buffered recent records per env (latest record per run). */
|
||||
replayMaxRunsPerEnv?: number;
|
||||
/** How long (ms) to keep an env subscribed + buffering after its last feed closes. 0 disables. */
|
||||
unsubscribeLingerMs?: number;
|
||||
/** Observability: a replay scan found candidates and delivered rows (or none survived). */
|
||||
onReplay?: (result: "delivered" | "empty") => void;
|
||||
/** Observability: a buffered record was evicted. `cap` evictions mean the env churns more
|
||||
* runs inside the window than the buffer holds (the replay guarantee is degrading). */
|
||||
onReplayEviction?: (reason: "cap" | "window") => void;
|
||||
/** Read-your-writes gate over the replica: delays wake-path hydrates until the replica
|
||||
* should have applied the change (record.updatedAtMs + lag + margin), and re-hydrates
|
||||
* rows the tripwire still finds stale. Omit to hydrate immediately (legacy behavior). */
|
||||
replicaLag?: ReplicaLagGate;
|
||||
};
|
||||
|
||||
export type ReplicaLagGate = {
|
||||
/** Current replica-lag estimate (ms). */
|
||||
getLagMs(): number;
|
||||
/** Feedback: a hydrate provably read at least this far behind the primary. */
|
||||
noteObservedLagMs(lagMs: number): void;
|
||||
/** Safety margin added on top of the estimate (clock skew + scheduling). */
|
||||
marginMs: number;
|
||||
/** Hard cap on any single gate delay — a sick replica degrades freshness, never liveness. */
|
||||
maxDelayMs: number;
|
||||
/** Re-hydrate attempts for rows that still read stale after the delay. */
|
||||
staleRetries: number;
|
||||
/** Observability: stale rows recovered by a retry, or delivered stale after exhausting them. */
|
||||
onStaleHydrate?: (outcome: "recovered" | "gave_up", runCount: number) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_REPLAY_WINDOW_MS = 2_000;
|
||||
const DEFAULT_REPLAY_MAX_RUNS_PER_ENV = 512;
|
||||
const DEFAULT_UNSUBSCRIBE_LINGER_MS = 5_000;
|
||||
|
||||
/** Handle a feed holds for the duration of one long-poll. */
|
||||
export type FeedRegistration = {
|
||||
/** Wait for the next batch matching this feed (or timeout/abort), with the matched runs
|
||||
* hydrated + serialized for this feed's columns. One wait active at a time. */
|
||||
waitForMatch(signal: AbortSignal | undefined, timeoutMs: number): Promise<WaitResult>;
|
||||
/** Deregister from the index; unsubscribes the env when the last feed leaves. */
|
||||
close(): void;
|
||||
/** False when this instance's env subscription is younger than the replay window, so a
|
||||
* change in the caller's inter-poll gap may have been missed (hop/cold start) — the
|
||||
* caller should resolve once instead of holding blind. */
|
||||
gapCovered: boolean;
|
||||
};
|
||||
|
||||
type Feed = {
|
||||
filter: FeedFilter;
|
||||
skipColumns: string[];
|
||||
columnSig: string;
|
||||
/** The currently-waiting poll's resolver (null between polls). */
|
||||
resolve: ((result: WaitResult) => void) | null;
|
||||
/** Buffered records at or before this timestamp have been replayed (or predate this feed). */
|
||||
replayCursorMs: number;
|
||||
};
|
||||
|
||||
type EnvState = {
|
||||
unsubscribe: () => void;
|
||||
feeds: Set<Feed>;
|
||||
byRunId: Map<string, Set<Feed>>;
|
||||
byTag: Map<string, Set<Feed>>;
|
||||
byBatchId: Map<string, Set<Feed>>;
|
||||
/** All tag feeds, for routing partial records (no tags) as hydrate-to-classify candidates. */
|
||||
tagFeeds: Set<Feed>;
|
||||
/** Tag feeds with no tag filter — they match every record but are unreachable via byTag. */
|
||||
unfilteredTagFeeds: Set<Feed>;
|
||||
/** When this env's channel subscription started (for the gap-coverage check). */
|
||||
subscribedAtMs: number;
|
||||
/** Latest record per run, insertion-ordered, for replaying inter-poll gaps to newly-armed feeds. */
|
||||
recent: Map<string, { record: ChangeRecord; receivedAtMs: number }>;
|
||||
/** Pending teardown while the env lingers with zero feeds. */
|
||||
lingerTimer?: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
timer.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
function addToIndex(index: Map<string, Set<Feed>>, key: string, feed: Feed) {
|
||||
let set = index.get(key);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
index.set(key, set);
|
||||
}
|
||||
set.add(feed);
|
||||
}
|
||||
|
||||
function removeFromIndex(index: Map<string, Set<Feed>>, key: string, feed: Feed) {
|
||||
const set = index.get(key);
|
||||
if (set) {
|
||||
set.delete(feed);
|
||||
if (set.size === 0) {
|
||||
index.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class EnvChangeRouter {
|
||||
readonly #envs = new Map<string, EnvState>();
|
||||
|
||||
constructor(private readonly options: EnvChangeRouterOptions) {}
|
||||
|
||||
register(
|
||||
environmentId: string,
|
||||
filter: FeedFilter,
|
||||
skipColumns: string[],
|
||||
opts?: {
|
||||
/** When the caller last received data for this connection. Bounds the replay to the
|
||||
* true inter-poll gap; older than the window can't be proven covered. */
|
||||
replaySinceMs?: number;
|
||||
}
|
||||
): FeedRegistration {
|
||||
const env = this.#ensureEnv(environmentId);
|
||||
const replayWindowMs = this.options.replayWindowMs ?? DEFAULT_REPLAY_WINDOW_MS;
|
||||
const now = Date.now();
|
||||
const windowFloorMs = now - replayWindowMs;
|
||||
const sinceMs = opts?.replaySinceMs ?? windowFloorMs;
|
||||
const feed: Feed = {
|
||||
filter,
|
||||
skipColumns,
|
||||
columnSig: skipColumns.length > 0 ? [...skipColumns].sort().join(",") : "",
|
||||
resolve: null,
|
||||
// First arm replays the caller's inter-poll gap; later arms only what arrived since.
|
||||
// The buffer only spans the window, so never rewind past it.
|
||||
replayCursorMs: Math.max(sinceMs, windowFloorMs),
|
||||
};
|
||||
|
||||
env.feeds.add(feed);
|
||||
this.#indexFeed(env, feed);
|
||||
|
||||
const waitForMatch = (signal: AbortSignal | undefined, timeoutMs: number) =>
|
||||
new Promise<WaitResult>((resolve) => {
|
||||
if (signal?.aborted) {
|
||||
resolve({ reason: "abort", rows: [] });
|
||||
return;
|
||||
}
|
||||
let settled = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const settle = (result: WaitResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
feed.resolve = null;
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
};
|
||||
feed.resolve = settle;
|
||||
timer = setTimeout(() => settle({ reason: "timeout", rows: [] }), timeoutMs);
|
||||
timer.unref?.();
|
||||
if (signal) {
|
||||
onAbort = () => settle({ reason: "abort", rows: [] });
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
// Deliver any buffered records this feed hasn't seen (catches changes that
|
||||
// landed while the caller was between polls).
|
||||
if (replayWindowMs > 0 && env.recent.size > 0) {
|
||||
this.#replayRecent(environmentId, env, feed).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to replay buffered records", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const close = () => {
|
||||
if (!env.feeds.has(feed)) {
|
||||
return;
|
||||
}
|
||||
env.feeds.delete(feed);
|
||||
this.#deindexFeed(env, feed);
|
||||
// Resolve any in-flight wait so the poll doesn't hang.
|
||||
feed.resolve?.({ reason: "abort", rows: [] });
|
||||
feed.resolve = null;
|
||||
if (env.feeds.size === 0) {
|
||||
this.#scheduleEnvTeardown(environmentId, env);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
waitForMatch,
|
||||
close,
|
||||
// Covered when this instance was already subscribed (and buffering) at the gap's
|
||||
// start, and the gap fits inside the buffer's window.
|
||||
gapCovered:
|
||||
replayWindowMs <= 0 || (env.subscribedAtMs <= sinceMs && sinceMs >= windowFloorMs),
|
||||
};
|
||||
}
|
||||
|
||||
/** Distinct environments currently routed (for metrics). */
|
||||
get activeEnvCount(): number {
|
||||
return this.#envs.size;
|
||||
}
|
||||
|
||||
/** Currently-held feeds by kind (for metrics) — the system's capacity unit. */
|
||||
get heldFeedCounts(): { run: number; tag: number; batch: number } {
|
||||
const counts = { run: 0, tag: 0, batch: 0 };
|
||||
for (const env of this.#envs.values()) {
|
||||
for (const feed of env.feeds) {
|
||||
counts[feed.filter.kind]++;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
#ensureEnv(environmentId: string): EnvState {
|
||||
const existing = this.#envs.get(environmentId);
|
||||
if (existing) {
|
||||
// A pending teardown is cancelled by new interest; the buffer survives the gap.
|
||||
if (existing.lingerTimer) {
|
||||
clearTimeout(existing.lingerTimer);
|
||||
existing.lingerTimer = undefined;
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
const env: EnvState = {
|
||||
unsubscribe: () => {},
|
||||
feeds: new Set(),
|
||||
byRunId: new Map(),
|
||||
byTag: new Map(),
|
||||
byBatchId: new Map(),
|
||||
tagFeeds: new Set(),
|
||||
unfilteredTagFeeds: new Set(),
|
||||
subscribedAtMs: Date.now(),
|
||||
recent: new Map(),
|
||||
};
|
||||
this.#envs.set(environmentId, env);
|
||||
env.unsubscribe = this.options.source.subscribeToEnv(environmentId, (records) => {
|
||||
this.#bufferRecent(env, records);
|
||||
// Fire-and-forget; catch hydrate failures here (unhandled rejection exits the process) — waiters time out into the backstop.
|
||||
this.#onBatch(environmentId, env, records).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to route a change batch", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
});
|
||||
return env;
|
||||
}
|
||||
|
||||
/** Keep the env subscribed + buffering for a linger after its last feed closes, so a
|
||||
* client's next poll (or another instance hop landing back here) can replay the gap. */
|
||||
#scheduleEnvTeardown(environmentId: string, env: EnvState) {
|
||||
const lingerMs = this.options.unsubscribeLingerMs ?? DEFAULT_UNSUBSCRIBE_LINGER_MS;
|
||||
if (lingerMs <= 0) {
|
||||
this.#envs.delete(environmentId);
|
||||
env.unsubscribe();
|
||||
return;
|
||||
}
|
||||
if (env.lingerTimer) {
|
||||
clearTimeout(env.lingerTimer);
|
||||
}
|
||||
env.lingerTimer = setTimeout(() => {
|
||||
if (env.feeds.size === 0) {
|
||||
this.#envs.delete(environmentId);
|
||||
env.unsubscribe();
|
||||
}
|
||||
}, lingerMs);
|
||||
env.lingerTimer.unref?.();
|
||||
}
|
||||
|
||||
/** Upsert the latest record per run (insertion-ordered) and prune to the window + cap. */
|
||||
#bufferRecent(env: EnvState, records: ChangeRecord[]) {
|
||||
const windowMs = this.options.replayWindowMs ?? DEFAULT_REPLAY_WINDOW_MS;
|
||||
if (windowMs <= 0) {
|
||||
return;
|
||||
}
|
||||
const maxRuns = this.options.replayMaxRunsPerEnv ?? DEFAULT_REPLAY_MAX_RUNS_PER_ENV;
|
||||
const now = Date.now();
|
||||
for (const record of records) {
|
||||
env.recent.delete(record.runId);
|
||||
env.recent.set(record.runId, { record, receivedAtMs: now });
|
||||
}
|
||||
const cutoff = now - windowMs;
|
||||
for (const [runId, entry] of env.recent) {
|
||||
if (entry.receivedAtMs >= cutoff && env.recent.size <= maxRuns) {
|
||||
break;
|
||||
}
|
||||
this.options.onReplayEviction?.(entry.receivedAtMs < cutoff ? "window" : "cap");
|
||||
env.recent.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a buffered record matches a feed's predicate (mirrors #onBatch's routing). */
|
||||
#recordMatchesFeed(record: ChangeRecord, feed: Feed): boolean {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
return record.runId === feed.filter.runId;
|
||||
case "batch":
|
||||
return record.batchId != null && record.batchId === feed.filter.batchId;
|
||||
case "tag": {
|
||||
const tags = feed.filter.tags;
|
||||
// Unfiltered feed matches everything; partial record (no tags) = hydrate-to-classify.
|
||||
if (tags.length === 0 || record.tags === undefined) {
|
||||
return true;
|
||||
}
|
||||
return record.tags.some((tag) => tags.includes(tag));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** How long to wait before hydrating so the replica has applied every change in the
|
||||
* batch: each record is safe at updatedAtMs + lag + margin (records without a watermark
|
||||
* anchor at now, degrading to a plain lag-sized delay). Capped — see ReplicaLagGate. */
|
||||
#gateDelayMs(records: ChangeRecord[]): number {
|
||||
const gate = this.options.replicaLag;
|
||||
if (!gate || records.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const now = Date.now();
|
||||
const lagMs = gate.getLagMs();
|
||||
let safeAtMs = 0;
|
||||
for (const record of records) {
|
||||
const anchorMs = record.updatedAtMs ?? now;
|
||||
safeAtMs = Math.max(safeAtMs, anchorMs + lagMs + gate.marginMs);
|
||||
}
|
||||
return Math.max(0, Math.min(safeAtMs - now, gate.maxDelayMs));
|
||||
}
|
||||
|
||||
/** Deliver buffered records newer than the feed's cursor through the normal
|
||||
* hydrate -> serialize -> settle pipeline. Already-seen rows diff to nothing downstream. */
|
||||
async #replayRecent(environmentId: string, env: EnvState, feed: Feed) {
|
||||
const cursor = feed.replayCursorMs;
|
||||
feed.replayCursorMs = Date.now();
|
||||
|
||||
const runIds: string[] = [];
|
||||
const candidateRecords: ChangeRecord[] = [];
|
||||
for (const [runId, entry] of env.recent) {
|
||||
if (entry.receivedAtMs > cursor && this.#recordMatchesFeed(entry.record, feed)) {
|
||||
runIds.push(runId);
|
||||
candidateRecords.push(entry.record);
|
||||
}
|
||||
}
|
||||
if (runIds.length === 0 || !feed.resolve) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Replayed records are usually past the lag window already (delay computes to 0); a
|
||||
// just-buffered one gets the same read-your-writes gate as the live path. No tripwire
|
||||
// here — a stale replay diffs to a re-emission on the next wake or backstop.
|
||||
const replayDelayMs = this.#gateDelayMs(candidateRecords);
|
||||
if (replayDelayMs > 0) {
|
||||
await sleepMs(replayDelayMs);
|
||||
if (!feed.resolve) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const hydrated = await this.options.hydrator.hydrateByIds(
|
||||
environmentId,
|
||||
runIds,
|
||||
feed.skipColumns
|
||||
);
|
||||
this.options.onHydrate?.(hydrated.length);
|
||||
|
||||
const rows: MatchedRow[] = [];
|
||||
for (const row of hydrated) {
|
||||
if (feed.filter.kind === "tag" && !this.#tagRowMatches(row, feed.filter)) {
|
||||
continue;
|
||||
}
|
||||
rows.push({ row, value: serializeRunRow(row, feed.skipColumns) });
|
||||
}
|
||||
|
||||
if (rows.length > 0 && feed.resolve) {
|
||||
this.options.onReplay?.("delivered");
|
||||
feed.resolve({ reason: "notify", rows });
|
||||
} else {
|
||||
this.options.onReplay?.("empty");
|
||||
}
|
||||
}
|
||||
|
||||
#indexFeed(env: EnvState, feed: Feed) {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
addToIndex(env.byRunId, feed.filter.runId, feed);
|
||||
break;
|
||||
case "batch":
|
||||
addToIndex(env.byBatchId, feed.filter.batchId, feed);
|
||||
break;
|
||||
case "tag":
|
||||
env.tagFeeds.add(feed);
|
||||
if (feed.filter.tags.length === 0) {
|
||||
env.unfilteredTagFeeds.add(feed);
|
||||
}
|
||||
for (const tag of feed.filter.tags) {
|
||||
addToIndex(env.byTag, tag, feed);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#deindexFeed(env: EnvState, feed: Feed) {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
removeFromIndex(env.byRunId, feed.filter.runId, feed);
|
||||
break;
|
||||
case "batch":
|
||||
removeFromIndex(env.byBatchId, feed.filter.batchId, feed);
|
||||
break;
|
||||
case "tag":
|
||||
env.tagFeeds.delete(feed);
|
||||
env.unfilteredTagFeeds.delete(feed);
|
||||
for (const tag of feed.filter.tags) {
|
||||
removeFromIndex(env.byTag, tag, feed);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async #onBatch(environmentId: string, env: EnvState, records: ChangeRecord[], attempt = 0) {
|
||||
// 0. Read-your-writes gate: wait out the replica's apply lag before hydrating, so the
|
||||
// rows we read contain the changes the records announce. Retry attempts were
|
||||
// scheduled with their own delay, so only the first pass gates here.
|
||||
if (attempt === 0) {
|
||||
const delayMs = this.#gateDelayMs(records);
|
||||
if (delayMs > 0) {
|
||||
await sleepMs(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Route each record to the held feeds it matches; collect matched runIds per feed.
|
||||
const matchedRunIdsByFeed = new Map<Feed, Set<string>>();
|
||||
const addMatch = (feed: Feed, runId: string) => {
|
||||
if (!feed.resolve) {
|
||||
// Feed isn't currently waiting (between polls). Drop — its backstop catches gaps.
|
||||
return;
|
||||
}
|
||||
let set = matchedRunIdsByFeed.get(feed);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
matchedRunIdsByFeed.set(feed, set);
|
||||
}
|
||||
set.add(runId);
|
||||
};
|
||||
|
||||
for (const record of records) {
|
||||
// run feeds: exact runId match.
|
||||
const runFeeds = env.byRunId.get(record.runId);
|
||||
if (runFeeds) {
|
||||
for (const feed of runFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
|
||||
// batch feeds: exact batchId match (only when the record carries one).
|
||||
if (record.batchId) {
|
||||
const batchFeeds = env.byBatchId.get(record.batchId);
|
||||
if (batchFeeds) {
|
||||
for (const feed of batchFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
|
||||
// tag feeds.
|
||||
if (record.tags !== undefined) {
|
||||
// Full record: prune via the tag index; only feeds whose filter intersects match.
|
||||
const seen = new Set<Feed>();
|
||||
for (const tag of record.tags) {
|
||||
const tagFeeds = env.byTag.get(tag);
|
||||
if (!tagFeeds) continue;
|
||||
for (const feed of tagFeeds) {
|
||||
if (seen.has(feed)) continue;
|
||||
seen.add(feed);
|
||||
addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
// Unfiltered tag feeds match every record but live outside the index.
|
||||
for (const feed of env.unfilteredTagFeeds) addMatch(feed, record.runId);
|
||||
} else {
|
||||
// Partial record (no membership data): route to every tag feed as a candidate to
|
||||
// hydrate-and-classify (rare; the publish side emits full records in practice).
|
||||
for (const feed of env.tagFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedRunIdsByFeed.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Batch-hydrate ONCE per column set, then 3. serialize ONCE per (runId, column set).
|
||||
const runIdsByColumnSig = new Map<string, { skipColumns: string[]; runIds: Set<string> }>();
|
||||
for (const [feed, runIds] of matchedRunIdsByFeed) {
|
||||
let group = runIdsByColumnSig.get(feed.columnSig);
|
||||
if (!group) {
|
||||
group = { skipColumns: feed.skipColumns, runIds: new Set() };
|
||||
runIdsByColumnSig.set(feed.columnSig, group);
|
||||
}
|
||||
for (const id of runIds) group.runIds.add(id);
|
||||
}
|
||||
|
||||
const hydratedByColumnSig = new Map<string, Map<string, MatchedRow>>();
|
||||
await Promise.all(
|
||||
[...runIdsByColumnSig.entries()].map(async ([columnSig, group]) => {
|
||||
const ids = [...group.runIds];
|
||||
const rows = await this.options.hydrator.hydrateByIds(
|
||||
environmentId,
|
||||
ids,
|
||||
group.skipColumns
|
||||
);
|
||||
this.options.onHydrate?.(rows.length);
|
||||
const map = new Map<string, MatchedRow>();
|
||||
for (const row of rows) {
|
||||
map.set(row.id, { row, value: serializeRunRow(row, group.skipColumns) });
|
||||
}
|
||||
hydratedByColumnSig.set(columnSig, map);
|
||||
})
|
||||
);
|
||||
|
||||
// 3.5 Stale tripwire: a watermarked record whose hydrated row is older (or missing —
|
||||
// the insert race) read a replica that hadn't applied the change. Withhold those
|
||||
// rows and re-hydrate shortly. Exhausting the retry budget delivers what we have
|
||||
// (liveness over freshness) — but a stale emission advances the feed's cursor, so
|
||||
// it ALSO schedules echo passes past the gate: re-hydrates flowing through normal
|
||||
// emission, where the working-set diff drops unchanged rows and emits the fresh
|
||||
// version once the replica catches up. The backstop stays the terminal net.
|
||||
// Each detection feeds the lag estimator.
|
||||
const gate = this.options.replicaLag;
|
||||
const isEchoPass = gate !== undefined && attempt > gate.staleRetries;
|
||||
const staleRunIds = gate
|
||||
? this.#detectStaleRuns(records, runIdsByColumnSig, hydratedByColumnSig)
|
||||
: new Set<string>();
|
||||
if (attempt > 0 && !isEchoPass) {
|
||||
const recovered = new Set(records.map((r) => r.runId)).size - staleRunIds.size;
|
||||
if (recovered > 0) {
|
||||
gate?.onStaleHydrate?.("recovered", recovered);
|
||||
}
|
||||
}
|
||||
if (staleRunIds.size > 0 && gate) {
|
||||
const staleRecords = records.filter((record) => staleRunIds.has(record.runId));
|
||||
// Re-buffer the withheld records so a feed that re-arms between now and the next
|
||||
// pass replays them instead of waiting for its backstop.
|
||||
this.#bufferRecent(env, staleRecords);
|
||||
if (attempt >= gate.staleRetries) {
|
||||
// Budget exhausted: deliver the stale rows below (liveness) — but a stale emission
|
||||
// advances the feed's cursor, so keep echoing re-hydrates through normal emission
|
||||
// (the working-set diff drops unchanged rows, emits the fresh version when the
|
||||
// replica catches up). Echoes stop once the change ages past the horizon; deeper
|
||||
// outages are the backstop's job.
|
||||
if (attempt === gate.staleRetries) {
|
||||
gate.onStaleHydrate?.("gave_up", staleRunIds.size);
|
||||
}
|
||||
staleRunIds.clear();
|
||||
}
|
||||
const echoHorizonMs = gate.maxDelayMs * 10;
|
||||
const newestWatermarkMs = Math.max(...staleRecords.map((record) => record.updatedAtMs ?? 0));
|
||||
const withinEchoHorizon = Date.now() - newestWatermarkMs < echoHorizonMs;
|
||||
if (attempt < gate.staleRetries || withinEchoHorizon) {
|
||||
const retryDelayMs = Math.max(
|
||||
25,
|
||||
Math.min(gate.getLagMs() + gate.marginMs, gate.maxDelayMs)
|
||||
);
|
||||
const timer = setTimeout(() => {
|
||||
this.#onBatch(environmentId, env, staleRecords, attempt + 1).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to re-hydrate stale rows", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
}, retryDelayMs);
|
||||
timer.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Assemble each feed's matched rows (post-filtering tag feeds against the
|
||||
// authoritative hydrated row) and resolve its pending wait.
|
||||
for (const [feed, runIds] of matchedRunIdsByFeed) {
|
||||
if (!feed.resolve) {
|
||||
continue; // stopped waiting while we hydrated; its next poll/backstop covers it
|
||||
}
|
||||
const hydrated = hydratedByColumnSig.get(feed.columnSig);
|
||||
if (!hydrated) continue;
|
||||
|
||||
const rows: MatchedRow[] = [];
|
||||
for (const runId of runIds) {
|
||||
if (staleRunIds.has(runId)) {
|
||||
continue; // withheld; the scheduled re-hydrate delivers the fresh version
|
||||
}
|
||||
const matched = hydrated.get(runId);
|
||||
if (!matched) continue; // run not found / left the table
|
||||
if (feed.filter.kind === "tag" && !this.#tagRowMatches(matched.row, feed.filter)) {
|
||||
continue; // re-confirm tags + createdAt floor against the authoritative row
|
||||
}
|
||||
rows.push(matched);
|
||||
}
|
||||
|
||||
if (rows.length > 0) {
|
||||
feed.resolve({ reason: "notify", rows });
|
||||
}
|
||||
// No surviving rows (e.g. a partial-record candidate that didn't actually match):
|
||||
// leave the feed waiting; nothing relevant changed for it.
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs whose hydrated row is provably behind its record's watermark (stale content),
|
||||
* or absent entirely despite a watermark (the insert hasn't applied). Records without
|
||||
* `updatedAtMs` can't be judged and always pass. */
|
||||
#detectStaleRuns(
|
||||
records: ChangeRecord[],
|
||||
runIdsByColumnSig: Map<string, { skipColumns: string[]; runIds: Set<string> }>,
|
||||
hydratedByColumnSig: Map<string, Map<string, MatchedRow>>
|
||||
): Set<string> {
|
||||
const gate = this.options.replicaLag;
|
||||
const stale = new Set<string>();
|
||||
if (!gate) {
|
||||
return stale;
|
||||
}
|
||||
const expectedByRunId = new Map<string, number>();
|
||||
for (const record of records) {
|
||||
if (record.updatedAtMs !== undefined) {
|
||||
const existing = expectedByRunId.get(record.runId);
|
||||
if (existing === undefined || record.updatedAtMs > existing) {
|
||||
expectedByRunId.set(record.runId, record.updatedAtMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (expectedByRunId.size === 0) {
|
||||
return stale;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const [columnSig, group] of runIdsByColumnSig) {
|
||||
const hydrated = hydratedByColumnSig.get(columnSig);
|
||||
for (const runId of group.runIds) {
|
||||
const expected = expectedByRunId.get(runId);
|
||||
if (expected === undefined || stale.has(runId)) {
|
||||
continue;
|
||||
}
|
||||
const matched = hydrated?.get(runId);
|
||||
if (!matched || matched.row.updatedAt.getTime() < expected) {
|
||||
stale.add(runId);
|
||||
gate.noteObservedLagMs(now - expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
|
||||
/** Authoritative re-check for tag feeds: the hydrated row carries ALL the filter's tags
|
||||
* (Electric's `runTags @> ARRAY[...]` semantics) and its createdAt is within the window. */
|
||||
#tagRowMatches(row: RealtimeRunRow, filter: Extract<FeedFilter, { kind: "tag" }>): boolean {
|
||||
if (
|
||||
filter.createdAtFloorMs !== undefined &&
|
||||
row.createdAt.getTime() < filter.createdAtFloorMs
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const rowTags = row.runTags ?? [];
|
||||
return filter.tags.every((tag) => rowTags.includes(tag));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user