chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Tiny in-process bounded TTL cache shared by the realtime feeds: entries expire after `ttlMs` (evicted on read),
|
||||
* and at-capacity writes sweep expired entries then drop the oldest. A stored `undefined` is indistinguishable from a miss (use `null` for absence).
|
||||
*/
|
||||
export class BoundedTtlCache<V> {
|
||||
readonly #entries = new Map<string, { value: V; expiresAt: number }>();
|
||||
|
||||
constructor(
|
||||
private readonly ttlMs: number,
|
||||
private readonly maxEntries: number
|
||||
) {}
|
||||
|
||||
get(key: string): V | undefined {
|
||||
const entry = this.#entries.get(key);
|
||||
if (!entry) {
|
||||
return undefined;
|
||||
}
|
||||
if (entry.expiresAt > Date.now()) {
|
||||
return entry.value;
|
||||
}
|
||||
// Evict on read so expired entries don't linger until the next at-capacity
|
||||
// sweep — important for read-heavy / low-churn caches (per-handle working sets).
|
||||
this.#entries.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
set(key: string, value: V): void {
|
||||
// Only run capacity eviction when inserting a NEW key — updating an existing key
|
||||
// doesn't grow the map, so it must never drop an unrelated live entry.
|
||||
if (!this.#entries.has(key) && this.#entries.size >= this.maxEntries) {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of this.#entries) {
|
||||
if (entry.expiresAt <= now) {
|
||||
this.#entries.delete(key);
|
||||
}
|
||||
}
|
||||
if (this.#entries.size >= this.maxEntries) {
|
||||
const oldest = this.#entries.keys().next().value;
|
||||
if (oldest !== undefined) {
|
||||
this.#entries.delete(oldest);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.#entries.set(key, { value, expiresAt: Date.now() + this.ttlMs });
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.#entries.size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { env } from "~/env.server";
|
||||
|
||||
/**
|
||||
* Canonical storage URI for a session's chat.agent snapshot. Stamped on
|
||||
* `Session.chatSnapshotStoragePath` at row creation so PUT/GET presigns
|
||||
* resolve to the same store even if `OBJECT_STORE_DEFAULT_PROTOCOL`
|
||||
* changes later.
|
||||
*/
|
||||
export function chatSnapshotStoragePathForSession(friendlyId: string): string {
|
||||
const path = `sessions/${friendlyId}/snapshot.json`;
|
||||
const protocol = env.OBJECT_STORE_DEFAULT_PROTOCOL;
|
||||
return protocol ? `${protocol}://${path}` : path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the storage key/URI a session's chat snapshot is written to and read
|
||||
* from. Single source of truth shared by every reader/writer so they all hit
|
||||
* the same object store:
|
||||
* - the SDK write + boot read (via the `snapshot-url` presign route), and
|
||||
* - the dashboard `SessionPresenter` (Agent/Session view).
|
||||
*
|
||||
* Prefers `chatSnapshotStoragePath` stamped at row creation (already
|
||||
* protocol-qualified, e.g. `s3://sessions/{id}/snapshot.json`), falling back to
|
||||
* recomputing it for sessions created before the column existed. Using a bare,
|
||||
* unqualified key here is the bug this guards against: the object store applies
|
||||
* `OBJECT_STORE_DEFAULT_PROTOCOL` to unprefixed keys on PUT but not on GET, so a
|
||||
* bare key can write to one store and read from another.
|
||||
*/
|
||||
export function chatSnapshotStorageKey(session: {
|
||||
friendlyId: string;
|
||||
chatSnapshotStoragePath: string | null;
|
||||
}): string {
|
||||
return session.chatSnapshotStoragePath ?? chatSnapshotStoragePathForSession(session.friendlyId);
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { type ClickHouse } from "@internal/clickhouse";
|
||||
import { type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
|
||||
import { type RunListFilter, type RunListResolver } from "./runReader.server";
|
||||
|
||||
export type ClickHouseRunListResolverOptions = {
|
||||
/** Resolves the per-organization ClickHouse client (multi-tenant routing). */
|
||||
getClickhouse: (organizationId: string) => Promise<ClickHouse>;
|
||||
prisma: PrismaClientOrTransaction;
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolves the realtime tag/list filter into matching run ids via ClickHouse `listRunIds` (filter-only;
|
||||
* rows hydrated from Postgres by id afterward). Tag matching is contains-ALL, byte-matching Electric's
|
||||
* `runTags @> ARRAY[...]` shape.
|
||||
*/
|
||||
export class ClickHouseRunListResolver implements RunListResolver {
|
||||
constructor(private readonly options: ClickHouseRunListResolverOptions) {}
|
||||
|
||||
async resolveMatchingRunIds(filter: RunListFilter): Promise<string[]> {
|
||||
const clickhouse = await this.options.getClickhouse(filter.organizationId);
|
||||
const repository = new RunsRepository({ clickhouse, prisma: this.options.prisma });
|
||||
|
||||
const { runIds } = await repository.listRunIds({
|
||||
organizationId: filter.organizationId,
|
||||
projectId: filter.projectId,
|
||||
environmentId: filter.environmentId,
|
||||
tags: filter.tags && filter.tags.length > 0 ? filter.tags : undefined,
|
||||
// Contains-ALL, matching the Electric shape's `runTags @> ARRAY[...]` semantics.
|
||||
tagsMatch: "all",
|
||||
batchId: filter.batchId,
|
||||
from: filter.createdAtAfter?.getTime(),
|
||||
page: { size: filter.limit },
|
||||
});
|
||||
|
||||
// listRunIds is keyset-paginated; runIds is already capped to page.size (= limit).
|
||||
return runIds;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Duration string parsing for stream-basin retention / delete-on-empty
|
||||
* configuration. Used by `streamBasinProvisioner` (to convert to S2's
|
||||
* integer-seconds wire format) and by `env.server.ts` (to validate
|
||||
* duration-shaped env vars at boot rather than at first use).
|
||||
*
|
||||
* Accepts the short forms (`7d`, `30d`, `365d`, `1h`, `90m`, `45s`,
|
||||
* `2w`, `1y`) and the human forms (`7days`, `1week`, `1year`).
|
||||
*/
|
||||
|
||||
const PATTERN =
|
||||
/^(\d+)\s*(s|sec|secs|seconds?|m|min|mins|minutes?|h|hour|hours?|d|day|days?|w|week|weeks?|y|year|years?)$/;
|
||||
|
||||
export function isValidDuration(input: string): boolean {
|
||||
return PATTERN.test(input.trim().toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a duration string into seconds. Throws on garbage so a
|
||||
* misconfigured env var fails loudly. Use {@link isValidDuration}
|
||||
* for non-throwing validation (e.g. inside a Zod `.refine()`).
|
||||
*/
|
||||
export function parseDuration(input: string): number {
|
||||
const trimmed = input.trim().toLowerCase();
|
||||
const match = trimmed.match(PATTERN);
|
||||
if (!match) {
|
||||
throw new Error(`Invalid duration string: ${input}`);
|
||||
}
|
||||
const value = parseInt(match[1]!, 10);
|
||||
const unit = match[2]!;
|
||||
const multiplier = /^s/.test(unit)
|
||||
? 1
|
||||
: /^m(?:in|ins|inute|inutes)?$/.test(unit)
|
||||
? 60
|
||||
: /^h/.test(unit)
|
||||
? 3600
|
||||
: /^d/.test(unit)
|
||||
? 86400
|
||||
: /^w/.test(unit)
|
||||
? 604800
|
||||
: /^y/.test(unit)
|
||||
? 31_536_000
|
||||
: NaN;
|
||||
if (!Number.isFinite(multiplier)) {
|
||||
throw new Error(`Invalid duration unit: ${unit}`);
|
||||
}
|
||||
return value * multiplier;
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/**
|
||||
* Pure (no DB/Redis/env) Electric HTTP shape-stream wire serializer, byte-faithful to what the
|
||||
* deployed `@electric-sql/client` (1.0.14 + 0.4.0) and the SDK's `SubscribeRunRawShape` expect.
|
||||
* Each column value is wire-encoded as a string (or null) decoded via the `electric-schema` header;
|
||||
* `up-to-date` is the only control message that makes the client emit, and re-sending a full row is idempotent.
|
||||
*/
|
||||
|
||||
export type ElectricColumnType =
|
||||
| "text"
|
||||
| "timestamp"
|
||||
| "int4"
|
||||
| "int8"
|
||||
| "float8"
|
||||
| "bool"
|
||||
| "jsonb";
|
||||
|
||||
type ElectricColumn = {
|
||||
name: string;
|
||||
type: ElectricColumnType;
|
||||
/** Array dimensionality. 1 => `type[]` (Postgres `{a,b}` literal). */
|
||||
dims?: number;
|
||||
/** Array columns only: true when the column has no SQL default, so an empty value emits `null` (not `{}`). Prisma erases this distinction, so we re-derive it here. */
|
||||
emptyArrayAsNull?: boolean;
|
||||
};
|
||||
|
||||
/** Columns the realtime run feed exposes; keep in sync with `DEFAULT_ELECTRIC_COLUMNS`. `type`/`dims` drive the schema header and value encoding. */
|
||||
export const RUN_ELECTRIC_COLUMNS: ReadonlyArray<ElectricColumn> = [
|
||||
{ name: "id", type: "text" },
|
||||
{ name: "taskIdentifier", type: "text" },
|
||||
{ name: "createdAt", type: "timestamp" },
|
||||
{ name: "updatedAt", type: "timestamp" },
|
||||
{ name: "startedAt", type: "timestamp" },
|
||||
{ name: "delayUntil", type: "timestamp" },
|
||||
{ name: "queuedAt", type: "timestamp" },
|
||||
{ name: "expiredAt", type: "timestamp" },
|
||||
{ name: "completedAt", type: "timestamp" },
|
||||
{ name: "friendlyId", type: "text" },
|
||||
{ name: "number", type: "int4" },
|
||||
{ name: "isTest", type: "bool" },
|
||||
{ name: "status", type: "text" },
|
||||
{ name: "usageDurationMs", type: "int4" },
|
||||
{ name: "costInCents", type: "float8" },
|
||||
{ name: "baseCostInCents", type: "float8" },
|
||||
{ name: "ttl", type: "text" },
|
||||
{ name: "payload", type: "text" },
|
||||
{ name: "payloadType", type: "text" },
|
||||
{ name: "metadata", type: "text" },
|
||||
{ name: "metadataType", type: "text" },
|
||||
{ name: "output", type: "text" },
|
||||
{ name: "outputType", type: "text" },
|
||||
{ name: "runTags", type: "text", dims: 1, emptyArrayAsNull: true },
|
||||
{ name: "error", type: "jsonb" },
|
||||
{ name: "realtimeStreams", type: "text", dims: 1 },
|
||||
];
|
||||
|
||||
/** Columns that can never be skipped via `skipColumns` (mirrors realtimeClient). */
|
||||
export const RESERVED_COLUMNS = ["id", "taskIdentifier", "friendlyId", "status", "createdAt"];
|
||||
|
||||
/** A single run hydrated for the realtime feed; structurally compatible with the `RunHydrator` Prisma `TaskRun` projection. */
|
||||
export type RealtimeRunRow = {
|
||||
id: string;
|
||||
taskIdentifier: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
startedAt: Date | null;
|
||||
delayUntil: Date | null;
|
||||
queuedAt: Date | null;
|
||||
expiredAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
friendlyId: string;
|
||||
number: number;
|
||||
isTest: boolean;
|
||||
status: string;
|
||||
usageDurationMs: number;
|
||||
costInCents: number;
|
||||
baseCostInCents: number;
|
||||
ttl: string | null;
|
||||
payload: string;
|
||||
payloadType: string;
|
||||
metadata: string | null;
|
||||
metadataType: string;
|
||||
output: string | null;
|
||||
outputType: string;
|
||||
runTags: string[];
|
||||
error: unknown;
|
||||
realtimeStreams: string[];
|
||||
};
|
||||
|
||||
type Operation = "insert" | "update" | "delete";
|
||||
|
||||
type ChangeMessage = {
|
||||
key: string;
|
||||
value: Record<string, string | null>;
|
||||
headers: { operation: Operation };
|
||||
};
|
||||
|
||||
type ControlMessage = {
|
||||
headers: { control: "up-to-date" | "must-refetch" };
|
||||
};
|
||||
|
||||
type ShapeMessage = ChangeMessage | ControlMessage;
|
||||
|
||||
const UP_TO_DATE: ControlMessage = { headers: { control: "up-to-date" } };
|
||||
|
||||
function effectiveSkipColumns(skipColumns: string[]): Set<string> {
|
||||
return new Set(skipColumns.filter((c) => c !== "" && !RESERVED_COLUMNS.includes(c)));
|
||||
}
|
||||
|
||||
function quoteArrayElement(value: string): string {
|
||||
return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
}
|
||||
|
||||
function pgArrayLiteral(values: unknown[]): string {
|
||||
if (values.length === 0) {
|
||||
return "{}";
|
||||
}
|
||||
return `{${values.map((v) => quoteArrayElement(String(v))).join(",")}}`;
|
||||
}
|
||||
|
||||
function serializeValue(value: unknown, column: ElectricColumn): string | null {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (column.dims && column.dims > 0) {
|
||||
if (!Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
// A no-default array column stores NULL when empty, so Electric emits `null`
|
||||
// (not `{}`); match that here since Prisma handed us `[]` for the NULL value.
|
||||
if (value.length === 0 && column.emptyArrayAsNull) {
|
||||
return null;
|
||||
}
|
||||
return pgArrayLiteral(value);
|
||||
}
|
||||
|
||||
switch (column.type) {
|
||||
case "bool":
|
||||
// Postgres text representation; the client's parseBool accepts "t"/"f".
|
||||
return value ? "t" : "f";
|
||||
case "timestamp":
|
||||
// The SDK's RawShapeDate appends "Z" before parsing, so we emit the ISO
|
||||
// string WITHOUT the trailing "Z".
|
||||
return value instanceof Date ? value.toISOString().slice(0, -1) : String(value);
|
||||
case "jsonb":
|
||||
return JSON.stringify(value);
|
||||
case "int4":
|
||||
case "int8":
|
||||
case "float8":
|
||||
case "text":
|
||||
default:
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/** The merge key the client uses to reassemble a row across insert/update cycles. */
|
||||
export function runShapeKey(runId: string): string {
|
||||
return `"public"."TaskRun"/"${runId}"`;
|
||||
}
|
||||
|
||||
/** Encode a single run row into the wire `value` object (column -> string|null). */
|
||||
export function serializeRunRow(
|
||||
row: RealtimeRunRow,
|
||||
skipColumns: string[] = []
|
||||
): Record<string, string | null> {
|
||||
const skip = effectiveSkipColumns(skipColumns);
|
||||
const value: Record<string, string | null> = {};
|
||||
|
||||
for (const column of RUN_ELECTRIC_COLUMNS) {
|
||||
if (skip.has(column.name)) {
|
||||
continue;
|
||||
}
|
||||
value[column.name] = serializeValue((row as Record<string, unknown>)[column.name], column);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/** The `electric-schema` response header value for the (optionally trimmed) column set. */
|
||||
export function buildElectricSchemaHeader(skipColumns: string[] = []): string {
|
||||
const skip = effectiveSkipColumns(skipColumns);
|
||||
const schema: Record<string, { type: string; dims?: number }> = {};
|
||||
|
||||
for (const column of RUN_ELECTRIC_COLUMNS) {
|
||||
if (skip.has(column.name)) {
|
||||
continue;
|
||||
}
|
||||
schema[column.name] = column.dims
|
||||
? { type: column.type, dims: column.dims }
|
||||
: { type: column.type };
|
||||
}
|
||||
|
||||
return JSON.stringify(schema);
|
||||
}
|
||||
|
||||
/** Initial snapshot body: an `insert` for the row (if present) then `up-to-date`; an absent row emits a bare `up-to-date` (empty shape). */
|
||||
export function buildSnapshotBody(row: RealtimeRunRow | null, skipColumns: string[] = []): string {
|
||||
const messages: ShapeMessage[] = [];
|
||||
if (row) {
|
||||
messages.push({
|
||||
key: runShapeKey(row.id),
|
||||
value: serializeRunRow(row, skipColumns),
|
||||
headers: { operation: "insert" },
|
||||
});
|
||||
}
|
||||
messages.push(UP_TO_DATE);
|
||||
return JSON.stringify(messages);
|
||||
}
|
||||
|
||||
/** Live body when the row advanced: a full-row `update` followed by `up-to-date`. */
|
||||
export function buildUpdateBody(row: RealtimeRunRow, skipColumns: string[] = []): string {
|
||||
const messages: ShapeMessage[] = [
|
||||
{
|
||||
key: runShapeKey(row.id),
|
||||
value: serializeRunRow(row, skipColumns),
|
||||
headers: { operation: "update" },
|
||||
},
|
||||
UP_TO_DATE,
|
||||
];
|
||||
return JSON.stringify(messages);
|
||||
}
|
||||
|
||||
/** Live body when nothing advanced: a bare `up-to-date` (no row emission). */
|
||||
export function buildUpToDateBody(): string {
|
||||
return JSON.stringify([UP_TO_DATE]);
|
||||
}
|
||||
|
||||
export type RowChange = { row: RealtimeRunRow; operation: "insert" | "update" };
|
||||
|
||||
/** Multi-row body for the tag-list feed: one change message per row then `up-to-date` (empty `changes` emits a bare `up-to-date`). */
|
||||
export function buildRowsBody(changes: RowChange[], skipColumns: string[] = []): string {
|
||||
const messages: ShapeMessage[] = changes.map((change) => ({
|
||||
key: runShapeKey(change.row.id),
|
||||
value: serializeRunRow(change.row, skipColumns),
|
||||
headers: { operation: change.operation },
|
||||
}));
|
||||
messages.push(UP_TO_DATE);
|
||||
return JSON.stringify(messages);
|
||||
}
|
||||
|
||||
/** A row change whose wire `value` was already serialized (once, shared across feeds by
|
||||
* the EnvChangeRouter); the per-feed `operation` is applied here. */
|
||||
export type SerializedRowChange = {
|
||||
runId: string;
|
||||
value: Record<string, string | null>;
|
||||
operation: "insert" | "update";
|
||||
};
|
||||
|
||||
/** Like `buildRowsBody`, but from values serialized once per (runId, columnSet) upstream,
|
||||
* so a run matching many feeds is serialized once and reused across their bodies. */
|
||||
export function buildRowsBodyFromSerialized(changes: SerializedRowChange[]): string {
|
||||
const messages: ShapeMessage[] = changes.map((change) => ({
|
||||
key: runShapeKey(change.runId),
|
||||
value: change.value,
|
||||
headers: { operation: change.operation },
|
||||
}));
|
||||
messages.push(UP_TO_DATE);
|
||||
return JSON.stringify(messages);
|
||||
}
|
||||
|
||||
export const INITIAL_OFFSET = "-1";
|
||||
|
||||
/** Opaque `<updatedAtMs>_<seq>` offset token (client `${number}_${number}` type); the first segment lets a live request detect whether the row advanced. */
|
||||
export function encodeOffset(updatedAtMs: number, seq: number): string {
|
||||
return `${Math.trunc(updatedAtMs)}_${Math.trunc(seq)}`;
|
||||
}
|
||||
|
||||
/** Extract the `updatedAt` epoch-ms a client last saw from its echoed offset. */
|
||||
export function parseOffsetUpdatedAtMs(offset: string | null | undefined): number {
|
||||
if (!offset) {
|
||||
return 0;
|
||||
}
|
||||
const [first] = offset.split("_");
|
||||
const value = Number(first);
|
||||
return Number.isFinite(value) && value > 0 ? value : 0;
|
||||
}
|
||||
|
||||
/** Mirror of realtimeClient's DEQUEUED->EXECUTING rewrite for non-current API versions. */
|
||||
export function rewriteBodyForLegacyApiVersion(body: string): string {
|
||||
return body.replace(/"status":"DEQUEUED"/g, '"status":"EXECUTING"');
|
||||
}
|
||||
@@ -0,0 +1,695 @@
|
||||
import { type ChangeRecord } from "./runChangeNotifier.server";
|
||||
import { type RealtimeRunRow, serializeRunRow } from "./electricStreamProtocol.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
/**
|
||||
* EnvChangeRouter — per-instance routing layer that fans one env's change stream out to the feeds it
|
||||
* matches. Owns one subscription per env (over the RunChangeNotifier) plus an inverted index of held
|
||||
* feeds, then per batch: routes via the index, batch-hydrates matched runs once per column set,
|
||||
* serializes each row's wire value once, and resolves each matched feed's pending wait. Stateless across reconnects.
|
||||
*/
|
||||
|
||||
export type WakeReason = "notify" | "timeout" | "abort";
|
||||
|
||||
/** A feed's membership predicate over the env stream. */
|
||||
export type FeedFilter =
|
||||
| { kind: "run"; runId: string }
|
||||
| { kind: "tag"; tags: string[]; createdAtFloorMs?: number }
|
||||
| { kind: "batch"; batchId: string };
|
||||
|
||||
/** A matched run handed to a feed: the hydrated row (for the feed's working-set diff) and
|
||||
* its wire `value` serialized once for this feed's column set (shared across feeds). */
|
||||
export type MatchedRow = { row: RealtimeRunRow; value: Record<string, string | null> };
|
||||
|
||||
export type WaitResult = { reason: WakeReason; rows: MatchedRow[] };
|
||||
|
||||
/** Minimal deps so the router is unit-testable without Redis/Postgres. */
|
||||
export interface EnvChangeSource {
|
||||
subscribeToEnv(environmentId: string, onBatch: (records: ChangeRecord[]) => void): () => void;
|
||||
}
|
||||
export interface RowHydrator {
|
||||
hydrateByIds(
|
||||
environmentId: string,
|
||||
ids: string[],
|
||||
skipColumns: string[]
|
||||
): Promise<RealtimeRunRow[]>;
|
||||
}
|
||||
|
||||
export type EnvChangeRouterOptions = {
|
||||
source: EnvChangeSource;
|
||||
hydrator: RowHydrator;
|
||||
/** Observability: a hydrate-by-id batch ran (count = runs hydrated this tick). */
|
||||
onHydrate?: (runCount: number) => void;
|
||||
/** How far back (ms) a newly-armed feed replays buffered records. 0 disables replay. */
|
||||
replayWindowMs?: number;
|
||||
/** Cap on buffered recent records per env (latest record per run). */
|
||||
replayMaxRunsPerEnv?: number;
|
||||
/** How long (ms) to keep an env subscribed + buffering after its last feed closes. 0 disables. */
|
||||
unsubscribeLingerMs?: number;
|
||||
/** Observability: a replay scan found candidates and delivered rows (or none survived). */
|
||||
onReplay?: (result: "delivered" | "empty") => void;
|
||||
/** Observability: a buffered record was evicted. `cap` evictions mean the env churns more
|
||||
* runs inside the window than the buffer holds (the replay guarantee is degrading). */
|
||||
onReplayEviction?: (reason: "cap" | "window") => void;
|
||||
/** Read-your-writes gate over the replica: delays wake-path hydrates until the replica
|
||||
* should have applied the change (record.updatedAtMs + lag + margin), and re-hydrates
|
||||
* rows the tripwire still finds stale. Omit to hydrate immediately (legacy behavior). */
|
||||
replicaLag?: ReplicaLagGate;
|
||||
};
|
||||
|
||||
export type ReplicaLagGate = {
|
||||
/** Current replica-lag estimate (ms). */
|
||||
getLagMs(): number;
|
||||
/** Feedback: a hydrate provably read at least this far behind the primary. */
|
||||
noteObservedLagMs(lagMs: number): void;
|
||||
/** Safety margin added on top of the estimate (clock skew + scheduling). */
|
||||
marginMs: number;
|
||||
/** Hard cap on any single gate delay — a sick replica degrades freshness, never liveness. */
|
||||
maxDelayMs: number;
|
||||
/** Re-hydrate attempts for rows that still read stale after the delay. */
|
||||
staleRetries: number;
|
||||
/** Observability: stale rows recovered by a retry, or delivered stale after exhausting them. */
|
||||
onStaleHydrate?: (outcome: "recovered" | "gave_up", runCount: number) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_REPLAY_WINDOW_MS = 2_000;
|
||||
const DEFAULT_REPLAY_MAX_RUNS_PER_ENV = 512;
|
||||
const DEFAULT_UNSUBSCRIBE_LINGER_MS = 5_000;
|
||||
|
||||
/** Handle a feed holds for the duration of one long-poll. */
|
||||
export type FeedRegistration = {
|
||||
/** Wait for the next batch matching this feed (or timeout/abort), with the matched runs
|
||||
* hydrated + serialized for this feed's columns. One wait active at a time. */
|
||||
waitForMatch(signal: AbortSignal | undefined, timeoutMs: number): Promise<WaitResult>;
|
||||
/** Deregister from the index; unsubscribes the env when the last feed leaves. */
|
||||
close(): void;
|
||||
/** False when this instance's env subscription is younger than the replay window, so a
|
||||
* change in the caller's inter-poll gap may have been missed (hop/cold start) — the
|
||||
* caller should resolve once instead of holding blind. */
|
||||
gapCovered: boolean;
|
||||
};
|
||||
|
||||
type Feed = {
|
||||
filter: FeedFilter;
|
||||
skipColumns: string[];
|
||||
columnSig: string;
|
||||
/** The currently-waiting poll's resolver (null between polls). */
|
||||
resolve: ((result: WaitResult) => void) | null;
|
||||
/** Buffered records at or before this timestamp have been replayed (or predate this feed). */
|
||||
replayCursorMs: number;
|
||||
};
|
||||
|
||||
type EnvState = {
|
||||
unsubscribe: () => void;
|
||||
feeds: Set<Feed>;
|
||||
byRunId: Map<string, Set<Feed>>;
|
||||
byTag: Map<string, Set<Feed>>;
|
||||
byBatchId: Map<string, Set<Feed>>;
|
||||
/** All tag feeds, for routing partial records (no tags) as hydrate-to-classify candidates. */
|
||||
tagFeeds: Set<Feed>;
|
||||
/** Tag feeds with no tag filter — they match every record but are unreachable via byTag. */
|
||||
unfilteredTagFeeds: Set<Feed>;
|
||||
/** When this env's channel subscription started (for the gap-coverage check). */
|
||||
subscribedAtMs: number;
|
||||
/** Latest record per run, insertion-ordered, for replaying inter-poll gaps to newly-armed feeds. */
|
||||
recent: Map<string, { record: ChangeRecord; receivedAtMs: number }>;
|
||||
/** Pending teardown while the env lingers with zero feeds. */
|
||||
lingerTimer?: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
timer.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
function addToIndex(index: Map<string, Set<Feed>>, key: string, feed: Feed) {
|
||||
let set = index.get(key);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
index.set(key, set);
|
||||
}
|
||||
set.add(feed);
|
||||
}
|
||||
|
||||
function removeFromIndex(index: Map<string, Set<Feed>>, key: string, feed: Feed) {
|
||||
const set = index.get(key);
|
||||
if (set) {
|
||||
set.delete(feed);
|
||||
if (set.size === 0) {
|
||||
index.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class EnvChangeRouter {
|
||||
readonly #envs = new Map<string, EnvState>();
|
||||
|
||||
constructor(private readonly options: EnvChangeRouterOptions) {}
|
||||
|
||||
register(
|
||||
environmentId: string,
|
||||
filter: FeedFilter,
|
||||
skipColumns: string[],
|
||||
opts?: {
|
||||
/** When the caller last received data for this connection. Bounds the replay to the
|
||||
* true inter-poll gap; older than the window can't be proven covered. */
|
||||
replaySinceMs?: number;
|
||||
}
|
||||
): FeedRegistration {
|
||||
const env = this.#ensureEnv(environmentId);
|
||||
const replayWindowMs = this.options.replayWindowMs ?? DEFAULT_REPLAY_WINDOW_MS;
|
||||
const now = Date.now();
|
||||
const windowFloorMs = now - replayWindowMs;
|
||||
const sinceMs = opts?.replaySinceMs ?? windowFloorMs;
|
||||
const feed: Feed = {
|
||||
filter,
|
||||
skipColumns,
|
||||
columnSig: skipColumns.length > 0 ? [...skipColumns].sort().join(",") : "",
|
||||
resolve: null,
|
||||
// First arm replays the caller's inter-poll gap; later arms only what arrived since.
|
||||
// The buffer only spans the window, so never rewind past it.
|
||||
replayCursorMs: Math.max(sinceMs, windowFloorMs),
|
||||
};
|
||||
|
||||
env.feeds.add(feed);
|
||||
this.#indexFeed(env, feed);
|
||||
|
||||
const waitForMatch = (signal: AbortSignal | undefined, timeoutMs: number) =>
|
||||
new Promise<WaitResult>((resolve) => {
|
||||
if (signal?.aborted) {
|
||||
resolve({ reason: "abort", rows: [] });
|
||||
return;
|
||||
}
|
||||
let settled = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const settle = (result: WaitResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
feed.resolve = null;
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
};
|
||||
feed.resolve = settle;
|
||||
timer = setTimeout(() => settle({ reason: "timeout", rows: [] }), timeoutMs);
|
||||
timer.unref?.();
|
||||
if (signal) {
|
||||
onAbort = () => settle({ reason: "abort", rows: [] });
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
// Deliver any buffered records this feed hasn't seen (catches changes that
|
||||
// landed while the caller was between polls).
|
||||
if (replayWindowMs > 0 && env.recent.size > 0) {
|
||||
this.#replayRecent(environmentId, env, feed).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to replay buffered records", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const close = () => {
|
||||
if (!env.feeds.has(feed)) {
|
||||
return;
|
||||
}
|
||||
env.feeds.delete(feed);
|
||||
this.#deindexFeed(env, feed);
|
||||
// Resolve any in-flight wait so the poll doesn't hang.
|
||||
feed.resolve?.({ reason: "abort", rows: [] });
|
||||
feed.resolve = null;
|
||||
if (env.feeds.size === 0) {
|
||||
this.#scheduleEnvTeardown(environmentId, env);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
waitForMatch,
|
||||
close,
|
||||
// Covered when this instance was already subscribed (and buffering) at the gap's
|
||||
// start, and the gap fits inside the buffer's window.
|
||||
gapCovered:
|
||||
replayWindowMs <= 0 || (env.subscribedAtMs <= sinceMs && sinceMs >= windowFloorMs),
|
||||
};
|
||||
}
|
||||
|
||||
/** Distinct environments currently routed (for metrics). */
|
||||
get activeEnvCount(): number {
|
||||
return this.#envs.size;
|
||||
}
|
||||
|
||||
/** Currently-held feeds by kind (for metrics) — the system's capacity unit. */
|
||||
get heldFeedCounts(): { run: number; tag: number; batch: number } {
|
||||
const counts = { run: 0, tag: 0, batch: 0 };
|
||||
for (const env of this.#envs.values()) {
|
||||
for (const feed of env.feeds) {
|
||||
counts[feed.filter.kind]++;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
#ensureEnv(environmentId: string): EnvState {
|
||||
const existing = this.#envs.get(environmentId);
|
||||
if (existing) {
|
||||
// A pending teardown is cancelled by new interest; the buffer survives the gap.
|
||||
if (existing.lingerTimer) {
|
||||
clearTimeout(existing.lingerTimer);
|
||||
existing.lingerTimer = undefined;
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
const env: EnvState = {
|
||||
unsubscribe: () => {},
|
||||
feeds: new Set(),
|
||||
byRunId: new Map(),
|
||||
byTag: new Map(),
|
||||
byBatchId: new Map(),
|
||||
tagFeeds: new Set(),
|
||||
unfilteredTagFeeds: new Set(),
|
||||
subscribedAtMs: Date.now(),
|
||||
recent: new Map(),
|
||||
};
|
||||
this.#envs.set(environmentId, env);
|
||||
env.unsubscribe = this.options.source.subscribeToEnv(environmentId, (records) => {
|
||||
this.#bufferRecent(env, records);
|
||||
// Fire-and-forget; catch hydrate failures here (unhandled rejection exits the process) — waiters time out into the backstop.
|
||||
this.#onBatch(environmentId, env, records).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to route a change batch", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
});
|
||||
return env;
|
||||
}
|
||||
|
||||
/** Keep the env subscribed + buffering for a linger after its last feed closes, so a
|
||||
* client's next poll (or another instance hop landing back here) can replay the gap. */
|
||||
#scheduleEnvTeardown(environmentId: string, env: EnvState) {
|
||||
const lingerMs = this.options.unsubscribeLingerMs ?? DEFAULT_UNSUBSCRIBE_LINGER_MS;
|
||||
if (lingerMs <= 0) {
|
||||
this.#envs.delete(environmentId);
|
||||
env.unsubscribe();
|
||||
return;
|
||||
}
|
||||
if (env.lingerTimer) {
|
||||
clearTimeout(env.lingerTimer);
|
||||
}
|
||||
env.lingerTimer = setTimeout(() => {
|
||||
if (env.feeds.size === 0) {
|
||||
this.#envs.delete(environmentId);
|
||||
env.unsubscribe();
|
||||
}
|
||||
}, lingerMs);
|
||||
env.lingerTimer.unref?.();
|
||||
}
|
||||
|
||||
/** Upsert the latest record per run (insertion-ordered) and prune to the window + cap. */
|
||||
#bufferRecent(env: EnvState, records: ChangeRecord[]) {
|
||||
const windowMs = this.options.replayWindowMs ?? DEFAULT_REPLAY_WINDOW_MS;
|
||||
if (windowMs <= 0) {
|
||||
return;
|
||||
}
|
||||
const maxRuns = this.options.replayMaxRunsPerEnv ?? DEFAULT_REPLAY_MAX_RUNS_PER_ENV;
|
||||
const now = Date.now();
|
||||
for (const record of records) {
|
||||
env.recent.delete(record.runId);
|
||||
env.recent.set(record.runId, { record, receivedAtMs: now });
|
||||
}
|
||||
const cutoff = now - windowMs;
|
||||
for (const [runId, entry] of env.recent) {
|
||||
if (entry.receivedAtMs >= cutoff && env.recent.size <= maxRuns) {
|
||||
break;
|
||||
}
|
||||
this.options.onReplayEviction?.(entry.receivedAtMs < cutoff ? "window" : "cap");
|
||||
env.recent.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a buffered record matches a feed's predicate (mirrors #onBatch's routing). */
|
||||
#recordMatchesFeed(record: ChangeRecord, feed: Feed): boolean {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
return record.runId === feed.filter.runId;
|
||||
case "batch":
|
||||
return record.batchId != null && record.batchId === feed.filter.batchId;
|
||||
case "tag": {
|
||||
const tags = feed.filter.tags;
|
||||
// Unfiltered feed matches everything; partial record (no tags) = hydrate-to-classify.
|
||||
if (tags.length === 0 || record.tags === undefined) {
|
||||
return true;
|
||||
}
|
||||
return record.tags.some((tag) => tags.includes(tag));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** How long to wait before hydrating so the replica has applied every change in the
|
||||
* batch: each record is safe at updatedAtMs + lag + margin (records without a watermark
|
||||
* anchor at now, degrading to a plain lag-sized delay). Capped — see ReplicaLagGate. */
|
||||
#gateDelayMs(records: ChangeRecord[]): number {
|
||||
const gate = this.options.replicaLag;
|
||||
if (!gate || records.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const now = Date.now();
|
||||
const lagMs = gate.getLagMs();
|
||||
let safeAtMs = 0;
|
||||
for (const record of records) {
|
||||
const anchorMs = record.updatedAtMs ?? now;
|
||||
safeAtMs = Math.max(safeAtMs, anchorMs + lagMs + gate.marginMs);
|
||||
}
|
||||
return Math.max(0, Math.min(safeAtMs - now, gate.maxDelayMs));
|
||||
}
|
||||
|
||||
/** Deliver buffered records newer than the feed's cursor through the normal
|
||||
* hydrate -> serialize -> settle pipeline. Already-seen rows diff to nothing downstream. */
|
||||
async #replayRecent(environmentId: string, env: EnvState, feed: Feed) {
|
||||
const cursor = feed.replayCursorMs;
|
||||
feed.replayCursorMs = Date.now();
|
||||
|
||||
const runIds: string[] = [];
|
||||
const candidateRecords: ChangeRecord[] = [];
|
||||
for (const [runId, entry] of env.recent) {
|
||||
if (entry.receivedAtMs > cursor && this.#recordMatchesFeed(entry.record, feed)) {
|
||||
runIds.push(runId);
|
||||
candidateRecords.push(entry.record);
|
||||
}
|
||||
}
|
||||
if (runIds.length === 0 || !feed.resolve) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Replayed records are usually past the lag window already (delay computes to 0); a
|
||||
// just-buffered one gets the same read-your-writes gate as the live path. No tripwire
|
||||
// here — a stale replay diffs to a re-emission on the next wake or backstop.
|
||||
const replayDelayMs = this.#gateDelayMs(candidateRecords);
|
||||
if (replayDelayMs > 0) {
|
||||
await sleepMs(replayDelayMs);
|
||||
if (!feed.resolve) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const hydrated = await this.options.hydrator.hydrateByIds(
|
||||
environmentId,
|
||||
runIds,
|
||||
feed.skipColumns
|
||||
);
|
||||
this.options.onHydrate?.(hydrated.length);
|
||||
|
||||
const rows: MatchedRow[] = [];
|
||||
for (const row of hydrated) {
|
||||
if (feed.filter.kind === "tag" && !this.#tagRowMatches(row, feed.filter)) {
|
||||
continue;
|
||||
}
|
||||
rows.push({ row, value: serializeRunRow(row, feed.skipColumns) });
|
||||
}
|
||||
|
||||
if (rows.length > 0 && feed.resolve) {
|
||||
this.options.onReplay?.("delivered");
|
||||
feed.resolve({ reason: "notify", rows });
|
||||
} else {
|
||||
this.options.onReplay?.("empty");
|
||||
}
|
||||
}
|
||||
|
||||
#indexFeed(env: EnvState, feed: Feed) {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
addToIndex(env.byRunId, feed.filter.runId, feed);
|
||||
break;
|
||||
case "batch":
|
||||
addToIndex(env.byBatchId, feed.filter.batchId, feed);
|
||||
break;
|
||||
case "tag":
|
||||
env.tagFeeds.add(feed);
|
||||
if (feed.filter.tags.length === 0) {
|
||||
env.unfilteredTagFeeds.add(feed);
|
||||
}
|
||||
for (const tag of feed.filter.tags) {
|
||||
addToIndex(env.byTag, tag, feed);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#deindexFeed(env: EnvState, feed: Feed) {
|
||||
switch (feed.filter.kind) {
|
||||
case "run":
|
||||
removeFromIndex(env.byRunId, feed.filter.runId, feed);
|
||||
break;
|
||||
case "batch":
|
||||
removeFromIndex(env.byBatchId, feed.filter.batchId, feed);
|
||||
break;
|
||||
case "tag":
|
||||
env.tagFeeds.delete(feed);
|
||||
env.unfilteredTagFeeds.delete(feed);
|
||||
for (const tag of feed.filter.tags) {
|
||||
removeFromIndex(env.byTag, tag, feed);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async #onBatch(environmentId: string, env: EnvState, records: ChangeRecord[], attempt = 0) {
|
||||
// 0. Read-your-writes gate: wait out the replica's apply lag before hydrating, so the
|
||||
// rows we read contain the changes the records announce. Retry attempts were
|
||||
// scheduled with their own delay, so only the first pass gates here.
|
||||
if (attempt === 0) {
|
||||
const delayMs = this.#gateDelayMs(records);
|
||||
if (delayMs > 0) {
|
||||
await sleepMs(delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Route each record to the held feeds it matches; collect matched runIds per feed.
|
||||
const matchedRunIdsByFeed = new Map<Feed, Set<string>>();
|
||||
const addMatch = (feed: Feed, runId: string) => {
|
||||
if (!feed.resolve) {
|
||||
// Feed isn't currently waiting (between polls). Drop — its backstop catches gaps.
|
||||
return;
|
||||
}
|
||||
let set = matchedRunIdsByFeed.get(feed);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
matchedRunIdsByFeed.set(feed, set);
|
||||
}
|
||||
set.add(runId);
|
||||
};
|
||||
|
||||
for (const record of records) {
|
||||
// run feeds: exact runId match.
|
||||
const runFeeds = env.byRunId.get(record.runId);
|
||||
if (runFeeds) {
|
||||
for (const feed of runFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
|
||||
// batch feeds: exact batchId match (only when the record carries one).
|
||||
if (record.batchId) {
|
||||
const batchFeeds = env.byBatchId.get(record.batchId);
|
||||
if (batchFeeds) {
|
||||
for (const feed of batchFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
|
||||
// tag feeds.
|
||||
if (record.tags !== undefined) {
|
||||
// Full record: prune via the tag index; only feeds whose filter intersects match.
|
||||
const seen = new Set<Feed>();
|
||||
for (const tag of record.tags) {
|
||||
const tagFeeds = env.byTag.get(tag);
|
||||
if (!tagFeeds) continue;
|
||||
for (const feed of tagFeeds) {
|
||||
if (seen.has(feed)) continue;
|
||||
seen.add(feed);
|
||||
addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
// Unfiltered tag feeds match every record but live outside the index.
|
||||
for (const feed of env.unfilteredTagFeeds) addMatch(feed, record.runId);
|
||||
} else {
|
||||
// Partial record (no membership data): route to every tag feed as a candidate to
|
||||
// hydrate-and-classify (rare; the publish side emits full records in practice).
|
||||
for (const feed of env.tagFeeds) addMatch(feed, record.runId);
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedRunIdsByFeed.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Batch-hydrate ONCE per column set, then 3. serialize ONCE per (runId, column set).
|
||||
const runIdsByColumnSig = new Map<string, { skipColumns: string[]; runIds: Set<string> }>();
|
||||
for (const [feed, runIds] of matchedRunIdsByFeed) {
|
||||
let group = runIdsByColumnSig.get(feed.columnSig);
|
||||
if (!group) {
|
||||
group = { skipColumns: feed.skipColumns, runIds: new Set() };
|
||||
runIdsByColumnSig.set(feed.columnSig, group);
|
||||
}
|
||||
for (const id of runIds) group.runIds.add(id);
|
||||
}
|
||||
|
||||
const hydratedByColumnSig = new Map<string, Map<string, MatchedRow>>();
|
||||
await Promise.all(
|
||||
[...runIdsByColumnSig.entries()].map(async ([columnSig, group]) => {
|
||||
const ids = [...group.runIds];
|
||||
const rows = await this.options.hydrator.hydrateByIds(
|
||||
environmentId,
|
||||
ids,
|
||||
group.skipColumns
|
||||
);
|
||||
this.options.onHydrate?.(rows.length);
|
||||
const map = new Map<string, MatchedRow>();
|
||||
for (const row of rows) {
|
||||
map.set(row.id, { row, value: serializeRunRow(row, group.skipColumns) });
|
||||
}
|
||||
hydratedByColumnSig.set(columnSig, map);
|
||||
})
|
||||
);
|
||||
|
||||
// 3.5 Stale tripwire: a watermarked record whose hydrated row is older (or missing —
|
||||
// the insert race) read a replica that hadn't applied the change. Withhold those
|
||||
// rows and re-hydrate shortly. Exhausting the retry budget delivers what we have
|
||||
// (liveness over freshness) — but a stale emission advances the feed's cursor, so
|
||||
// it ALSO schedules echo passes past the gate: re-hydrates flowing through normal
|
||||
// emission, where the working-set diff drops unchanged rows and emits the fresh
|
||||
// version once the replica catches up. The backstop stays the terminal net.
|
||||
// Each detection feeds the lag estimator.
|
||||
const gate = this.options.replicaLag;
|
||||
const isEchoPass = gate !== undefined && attempt > gate.staleRetries;
|
||||
const staleRunIds = gate
|
||||
? this.#detectStaleRuns(records, runIdsByColumnSig, hydratedByColumnSig)
|
||||
: new Set<string>();
|
||||
if (attempt > 0 && !isEchoPass) {
|
||||
const recovered = new Set(records.map((r) => r.runId)).size - staleRunIds.size;
|
||||
if (recovered > 0) {
|
||||
gate?.onStaleHydrate?.("recovered", recovered);
|
||||
}
|
||||
}
|
||||
if (staleRunIds.size > 0 && gate) {
|
||||
const staleRecords = records.filter((record) => staleRunIds.has(record.runId));
|
||||
// Re-buffer the withheld records so a feed that re-arms between now and the next
|
||||
// pass replays them instead of waiting for its backstop.
|
||||
this.#bufferRecent(env, staleRecords);
|
||||
if (attempt >= gate.staleRetries) {
|
||||
// Budget exhausted: deliver the stale rows below (liveness) — but a stale emission
|
||||
// advances the feed's cursor, so keep echoing re-hydrates through normal emission
|
||||
// (the working-set diff drops unchanged rows, emits the fresh version when the
|
||||
// replica catches up). Echoes stop once the change ages past the horizon; deeper
|
||||
// outages are the backstop's job.
|
||||
if (attempt === gate.staleRetries) {
|
||||
gate.onStaleHydrate?.("gave_up", staleRunIds.size);
|
||||
}
|
||||
staleRunIds.clear();
|
||||
}
|
||||
const echoHorizonMs = gate.maxDelayMs * 10;
|
||||
const newestWatermarkMs = Math.max(...staleRecords.map((record) => record.updatedAtMs ?? 0));
|
||||
const withinEchoHorizon = Date.now() - newestWatermarkMs < echoHorizonMs;
|
||||
if (attempt < gate.staleRetries || withinEchoHorizon) {
|
||||
const retryDelayMs = Math.max(
|
||||
25,
|
||||
Math.min(gate.getLagMs() + gate.marginMs, gate.maxDelayMs)
|
||||
);
|
||||
const timer = setTimeout(() => {
|
||||
this.#onBatch(environmentId, env, staleRecords, attempt + 1).catch((error) => {
|
||||
logger.error("[envChangeRouter] failed to re-hydrate stale rows", {
|
||||
environmentId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
}, retryDelayMs);
|
||||
timer.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Assemble each feed's matched rows (post-filtering tag feeds against the
|
||||
// authoritative hydrated row) and resolve its pending wait.
|
||||
for (const [feed, runIds] of matchedRunIdsByFeed) {
|
||||
if (!feed.resolve) {
|
||||
continue; // stopped waiting while we hydrated; its next poll/backstop covers it
|
||||
}
|
||||
const hydrated = hydratedByColumnSig.get(feed.columnSig);
|
||||
if (!hydrated) continue;
|
||||
|
||||
const rows: MatchedRow[] = [];
|
||||
for (const runId of runIds) {
|
||||
if (staleRunIds.has(runId)) {
|
||||
continue; // withheld; the scheduled re-hydrate delivers the fresh version
|
||||
}
|
||||
const matched = hydrated.get(runId);
|
||||
if (!matched) continue; // run not found / left the table
|
||||
if (feed.filter.kind === "tag" && !this.#tagRowMatches(matched.row, feed.filter)) {
|
||||
continue; // re-confirm tags + createdAt floor against the authoritative row
|
||||
}
|
||||
rows.push(matched);
|
||||
}
|
||||
|
||||
if (rows.length > 0) {
|
||||
feed.resolve({ reason: "notify", rows });
|
||||
}
|
||||
// No surviving rows (e.g. a partial-record candidate that didn't actually match):
|
||||
// leave the feed waiting; nothing relevant changed for it.
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs whose hydrated row is provably behind its record's watermark (stale content),
|
||||
* or absent entirely despite a watermark (the insert hasn't applied). Records without
|
||||
* `updatedAtMs` can't be judged and always pass. */
|
||||
#detectStaleRuns(
|
||||
records: ChangeRecord[],
|
||||
runIdsByColumnSig: Map<string, { skipColumns: string[]; runIds: Set<string> }>,
|
||||
hydratedByColumnSig: Map<string, Map<string, MatchedRow>>
|
||||
): Set<string> {
|
||||
const gate = this.options.replicaLag;
|
||||
const stale = new Set<string>();
|
||||
if (!gate) {
|
||||
return stale;
|
||||
}
|
||||
const expectedByRunId = new Map<string, number>();
|
||||
for (const record of records) {
|
||||
if (record.updatedAtMs !== undefined) {
|
||||
const existing = expectedByRunId.get(record.runId);
|
||||
if (existing === undefined || record.updatedAtMs > existing) {
|
||||
expectedByRunId.set(record.runId, record.updatedAtMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (expectedByRunId.size === 0) {
|
||||
return stale;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const [columnSig, group] of runIdsByColumnSig) {
|
||||
const hydrated = hydratedByColumnSig.get(columnSig);
|
||||
for (const runId of group.runIds) {
|
||||
const expected = expectedByRunId.get(runId);
|
||||
if (expected === undefined || stale.has(runId)) {
|
||||
continue;
|
||||
}
|
||||
const matched = hydrated?.get(runId);
|
||||
if (!matched || matched.row.updatedAt.getTime() < expected) {
|
||||
stale.add(runId);
|
||||
gate.noteObservedLagMs(now - expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
|
||||
/** Authoritative re-check for tag feeds: the hydrated row carries ALL the filter's tags
|
||||
* (Electric's `runTags @> ARRAY[...]` semantics) and its createdAt is within the window. */
|
||||
#tagRowMatches(row: RealtimeRunRow, filter: Extract<FeedFilter, { kind: "tag" }>): boolean {
|
||||
if (
|
||||
filter.createdAtFloorMs !== undefined &&
|
||||
row.createdAt.getTime() < filter.createdAtFloorMs
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const rowTags = row.runTags ?? [];
|
||||
return filter.tags.every((tag) => rowTags.includes(tag));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import { validateJWT, type ValidationResult } from "@trigger.dev/core/v3/jwt";
|
||||
import { $replica } from "~/db.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import type { AuthenticatedEnvironment } from "../apiAuth.server";
|
||||
|
||||
export type ValidatePublicJwtKeySuccess = {
|
||||
ok: true;
|
||||
environment: AuthenticatedEnvironment;
|
||||
claims: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type ValidatePublicJwtKeyError = {
|
||||
ok: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type ValidatePublicJwtKeyResult = ValidatePublicJwtKeySuccess | ValidatePublicJwtKeyError;
|
||||
|
||||
export async function validatePublicJwtKey(token: string): Promise<ValidatePublicJwtKeyResult> {
|
||||
// Get the sub claim from the token
|
||||
// Use the sub claim to find the environment
|
||||
// Validate the token against the environment.apiKey
|
||||
// Once that's done, return the environment and the claims
|
||||
const sub = extractJWTSub(token);
|
||||
|
||||
if (!sub) {
|
||||
return { ok: false, error: "Invalid Public Access Token, missing subject." };
|
||||
}
|
||||
|
||||
const environment = await findEnvironmentById(sub);
|
||||
|
||||
if (!environment) {
|
||||
return { ok: false, error: "Invalid Public Access Token, environment not found." };
|
||||
}
|
||||
|
||||
let result = await validateJWT(
|
||||
token,
|
||||
environment.parentEnvironment?.apiKey ?? environment.apiKey
|
||||
);
|
||||
|
||||
// PATs are signed with the env's apiKey at mint time. If the env's apiKey
|
||||
// has since been rotated, signature verification fails against the current
|
||||
// key — fall back to any RevokedApiKey rows still in their grace window.
|
||||
// Only run this query on the failure path so the success path is unchanged.
|
||||
if (!result.ok) {
|
||||
result = await validateAgainstRevokedApiKeys(
|
||||
token,
|
||||
environment.parentEnvironment?.id ?? environment.id,
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
if (!result.ok) {
|
||||
switch (result.code) {
|
||||
case "ERR_JWT_EXPIRED": {
|
||||
return {
|
||||
ok: false,
|
||||
error:
|
||||
"Public Access Token has expired. See https://trigger.dev/docs/frontend/overview#authentication for more information.",
|
||||
};
|
||||
}
|
||||
case "ERR_JWT_CLAIM_INVALID": {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Public Access Token is invalid: ${result.error}. See https://trigger.dev/docs/frontend/overview#authentication for more information.`,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return {
|
||||
ok: false,
|
||||
error:
|
||||
"Public Access Token is invalid. See https://trigger.dev/docs/frontend/overview#authentication for more information.",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
environment,
|
||||
claims: result.payload,
|
||||
};
|
||||
}
|
||||
|
||||
async function validateAgainstRevokedApiKeys(
|
||||
token: string,
|
||||
signingEnvironmentId: string,
|
||||
primaryResult: ValidationResult
|
||||
): Promise<ValidationResult> {
|
||||
const revokedApiKeys = await $replica.revokedApiKey.findMany({
|
||||
where: {
|
||||
runtimeEnvironmentId: signingEnvironmentId,
|
||||
expiresAt: { gt: new Date() },
|
||||
},
|
||||
select: { apiKey: true },
|
||||
});
|
||||
|
||||
for (const { apiKey } of revokedApiKeys) {
|
||||
const fallbackResult = await validateJWT(token, apiKey);
|
||||
if (fallbackResult.ok) {
|
||||
return fallbackResult;
|
||||
}
|
||||
}
|
||||
|
||||
return primaryResult;
|
||||
}
|
||||
|
||||
export function isPublicJWT(token: string): boolean {
|
||||
// Split the token
|
||||
const parts = token.split(".");
|
||||
if (parts.length !== 3) return false;
|
||||
|
||||
try {
|
||||
// Decode the payload (second part)
|
||||
const payload = JSON.parse(decodeBase64Url(parts[1]));
|
||||
|
||||
if (payload === null || typeof payload !== "object") return false;
|
||||
|
||||
// Check for the pub: true claim
|
||||
return "pub" in payload && payload.pub === true;
|
||||
} catch (_error) {
|
||||
// If there's any error in decoding or parsing, it's not a valid JWT
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractJwtSigningSecretKey(environment: AuthenticatedEnvironment) {
|
||||
return environment.parentEnvironment?.apiKey ?? environment.apiKey;
|
||||
}
|
||||
|
||||
function extractJWTSub(token: string): string | undefined {
|
||||
// Split the token
|
||||
const parts = token.split(".");
|
||||
if (parts.length !== 3) return;
|
||||
|
||||
try {
|
||||
// Decode the payload (second part)
|
||||
const payload = JSON.parse(decodeBase64Url(parts[1]));
|
||||
|
||||
if (payload === null || typeof payload !== "object") return;
|
||||
|
||||
// Check for the pub: true claim
|
||||
return "sub" in payload && typeof payload.sub === "string" ? payload.sub : undefined;
|
||||
} catch (_error) {
|
||||
// If there's any error in decoding or parsing, it's not a valid JWT
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeBase64Url(str: string): string {
|
||||
// Replace URL-safe characters and add padding
|
||||
str = str.replace(/-/g, "+").replace(/_/g, "/");
|
||||
switch (str.length % 4) {
|
||||
case 2:
|
||||
str += "==";
|
||||
break;
|
||||
case 3:
|
||||
str += "=";
|
||||
break;
|
||||
}
|
||||
|
||||
// Decode using Node.js Buffer
|
||||
return Buffer.from(str, "base64").toString("utf8");
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { generateJWT as internal_generateJWT } from "@trigger.dev/core/v3";
|
||||
import { extractJwtSigningSecretKey } from "./jwtAuth.server";
|
||||
|
||||
type Environment = Parameters<typeof extractJwtSigningSecretKey>[0];
|
||||
|
||||
export type MintRunTokenOptions = {
|
||||
/** Include the input-stream write scope (needed for steering messages from the playground). */
|
||||
includeInputStreamWrite?: boolean;
|
||||
/** Token expiration. Defaults to "1h". */
|
||||
expirationTime?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Mint a run-scoped public access token (JWT) for browser subscription to a
|
||||
* run's realtime streams.
|
||||
*
|
||||
* Used by:
|
||||
* - The playground action to give a freshly triggered chat session a token.
|
||||
* - The run details page to let the agent view subscribe to the chat stream
|
||||
* of an existing run (read-only).
|
||||
*/
|
||||
export async function mintRunToken(
|
||||
environment: Environment,
|
||||
runFriendlyId: string,
|
||||
options: MintRunTokenOptions = {}
|
||||
): Promise<string> {
|
||||
const scopes = [`read:runs:${runFriendlyId}`];
|
||||
if (options.includeInputStreamWrite) {
|
||||
scopes.push(`write:inputStreams:${runFriendlyId}`);
|
||||
}
|
||||
|
||||
return internal_generateJWT({
|
||||
secretKey: extractJwtSigningSecretKey(environment),
|
||||
payload: {
|
||||
sub: environment.id,
|
||||
pub: true,
|
||||
scopes,
|
||||
},
|
||||
expirationTime: options.expirationTime ?? "1h",
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { generateJWT as internal_generateJWT } from "@trigger.dev/core/v3";
|
||||
import { extractJwtSigningSecretKey } from "./jwtAuth.server";
|
||||
|
||||
type Environment = Parameters<typeof extractJwtSigningSecretKey>[0];
|
||||
|
||||
export type MintSessionTokenOptions = {
|
||||
/** Token expiration. Defaults to "1h". */
|
||||
expirationTime?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Mint a session-scoped public access token (JWT) covering both `.in`
|
||||
* append and `.out` subscribe for a session's realtime channels.
|
||||
*
|
||||
* Returned by `POST /api/v1/sessions` so the browser holds a single
|
||||
* long-lived token that survives across runs (sessions outlive any
|
||||
* single run). Includes both read and write scopes since the transport
|
||||
* needs both: read for SSE subscribe on `.out`, write for `.in` appends
|
||||
* (`stop`, follow-up messages, action chunks).
|
||||
*/
|
||||
export async function mintSessionToken(
|
||||
environment: Environment,
|
||||
sessionAddressingKey: string,
|
||||
options: MintSessionTokenOptions = {}
|
||||
): Promise<string> {
|
||||
const scopes = [
|
||||
`read:sessions:${sessionAddressingKey}`,
|
||||
`write:sessions:${sessionAddressingKey}`,
|
||||
];
|
||||
|
||||
return internal_generateJWT({
|
||||
secretKey: extractJwtSigningSecretKey(environment),
|
||||
payload: {
|
||||
sub: environment.id,
|
||||
pub: true,
|
||||
scopes,
|
||||
},
|
||||
expirationTime: options.expirationTime ?? "1h",
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,272 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
import { $replica } from "~/db.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { env } from "~/env.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { getCachedLimit } from "../platform.v3.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { ClickHouseRunListResolver } from "./clickHouseRunListResolver.server";
|
||||
import { EnvChangeRouter, type EnvChangeSource } from "./envChangeRouter.server";
|
||||
import { NativeRealtimeClient } from "./nativeRealtimeClient.server";
|
||||
import { RealtimeConcurrencyLimiter } from "./realtimeConcurrencyLimiter.server";
|
||||
import { getRunChangeNotifier } from "./runChangeNotifierInstance.server";
|
||||
import { RedisReplayCursorStore } from "./replayCursorStore.server";
|
||||
import { createPostgresReplicaLagSource, ReplicaLagEstimator } from "./replicaLagEstimator.server";
|
||||
import { RunHydrator } from "./runReader.server";
|
||||
|
||||
// Process-singleton wiring for the native realtime client; only constructed when a
|
||||
// request actually routes to it, so a disabled webapp never instantiates it.
|
||||
function initializeNativeRealtimeClient(): NativeRealtimeClient {
|
||||
const meter = getMeter("realtime-native");
|
||||
|
||||
const wakeups = meter.createCounter("realtime_native.wakeups", {
|
||||
description:
|
||||
"Live realtime wakeups by reason. A rising 'timeout' share suggests a write site is missing its publishChangeRecord delegate.",
|
||||
});
|
||||
|
||||
const runSetResolves = meter.createCounter("realtime_native.runset_resolves", {
|
||||
description:
|
||||
"Multi-run (tag-list/batch) resolve+hydrate outcomes. 'hit'/'coalesced' vs 'miss' shows how effectively concurrent same-filter feeds share a single ClickHouse + Postgres query.",
|
||||
});
|
||||
|
||||
const runSetQueryMs = meter.createHistogram("realtime_native.runset_query_ms", {
|
||||
description: "Latency of the multi-run resolve (ClickHouse) and hydrate (Postgres) stages.",
|
||||
unit: "ms",
|
||||
});
|
||||
|
||||
const livePollPaths = meter.createCounter("realtime_native.live_polls", {
|
||||
description:
|
||||
"How live polls resolved. 'fast-hydrate' = router wake with rows hydrated by id (no ClickHouse); 'full-resolve' = backstop; 'cold-resolve' = fresh env subscription probed once.",
|
||||
});
|
||||
|
||||
const routerHydrates = meter.createCounter("realtime_native.router_hydrated_runs", {
|
||||
description:
|
||||
"Runs hydrated by the EnvChangeRouter's batch-hydrate (one query per column set per wake, shared across all feeds matching the same run).",
|
||||
});
|
||||
|
||||
const resolveAdmissionWaits = meter.createCounter("realtime_native.resolve_admission_waits", {
|
||||
description:
|
||||
"Fresh ClickHouse resolves that had to queue for an admission permit. A rising count means a distinct-filter reconnect stampede is being throttled (the gate is doing its job).",
|
||||
});
|
||||
|
||||
const replays = meter.createCounter("realtime_native.replays", {
|
||||
description:
|
||||
"Buffered change records replayed to a newly-armed feed (inter-poll gap recovery). 'delivered' = rows reached the feed; 'empty' = candidates hydrated but none survived the filter/diff.",
|
||||
});
|
||||
|
||||
const replayEvictions = meter.createCounter("realtime_native.replay_evictions", {
|
||||
description:
|
||||
"Replay-buffer evictions. 'window' expiry is normal; 'cap' means an env churns more runs inside the window than the buffer holds (replay guarantee degrading — retune the knobs).",
|
||||
});
|
||||
|
||||
const deliveryLagMs = meter.createHistogram("realtime_native.delivery_lag_ms", {
|
||||
description:
|
||||
"Live emissions: now minus the newest emitted row's updatedAt (PG clock vs app clock, so approximate). The end-to-end delivery SLI — a p99 near the backstop hold means wakes are being missed.",
|
||||
unit: "ms",
|
||||
});
|
||||
|
||||
const emittedRows = meter.createHistogram("realtime_native.emitted_rows", {
|
||||
description:
|
||||
"Rows per live emission. Deltas should be small; a fat tail means working-set/offset-floor fallbacks are re-emitting full sets.",
|
||||
unit: "rows",
|
||||
});
|
||||
|
||||
const backstops = meter.createCounter("realtime_native.backstops", {
|
||||
description:
|
||||
"Backstop full resolves by outcome. 'empty' is normal idle behavior; sustained 'delivered' means the notify/replay path missed changes — alert on it.",
|
||||
});
|
||||
|
||||
const concurrencyRejections = meter.createCounter("realtime_native.concurrency_rejections", {
|
||||
description: "Polls rejected (429) by the per-env concurrency limiter.",
|
||||
});
|
||||
|
||||
const replayCursorOps = meter.createCounter("realtime_native.replay_cursor_ops", {
|
||||
description:
|
||||
"Shared replay-cursor store operations by outcome. Errors degrade hops to cold resolves (watch live_polls{path='cold-resolve'} rise with them), never failed polls.",
|
||||
});
|
||||
|
||||
const staleHydrates = meter.createCounter("realtime_native.stale_hydrates", {
|
||||
description:
|
||||
"Wake hydrates the read-your-writes tripwire caught reading behind the publish. 'recovered' = a retry delivered the fresh row; sustained 'gave_up' means replica lag is outrunning the retry budget.",
|
||||
});
|
||||
|
||||
const limiter = new RealtimeConcurrencyLimiter({
|
||||
keyPrefix: "tr:realtime:native:concurrency",
|
||||
redis: {
|
||||
port: env.RATE_LIMIT_REDIS_PORT,
|
||||
host: env.RATE_LIMIT_REDIS_HOST,
|
||||
username: env.RATE_LIMIT_REDIS_USERNAME,
|
||||
password: env.RATE_LIMIT_REDIS_PASSWORD,
|
||||
tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
},
|
||||
});
|
||||
|
||||
// Fleet-shared replay cursors (one timestamp per connection) on the same Redis as the
|
||||
// change channel, so a load-balancer hop reads the connection's true inter-poll gap.
|
||||
const replayCursorStore =
|
||||
env.REALTIME_BACKEND_NATIVE_SHARED_REPLAY_CURSORS === "1"
|
||||
? new RedisReplayCursorStore({
|
||||
redis: {
|
||||
host: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_HOST,
|
||||
port: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_PORT,
|
||||
username: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_USERNAME,
|
||||
password: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_PASSWORD,
|
||||
tlsDisabled: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
},
|
||||
ttlMs: env.REALTIME_BACKEND_NATIVE_WORKING_SET_TTL_MS,
|
||||
onResult: (op, ok) => replayCursorOps.add(1, { op, result: ok ? "ok" : "error" }),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
// One RunHydrator shared by the router and the client, so its single-flight + short-TTL cache covers both.
|
||||
const runReader = new RunHydrator({
|
||||
replica: $replica,
|
||||
runStore,
|
||||
cacheTtlMs: env.REALTIME_BACKEND_NATIVE_RUN_CACHE_TTL_MS,
|
||||
maxCacheEntries: env.REALTIME_BACKEND_NATIVE_RUN_CACHE_MAX_ENTRIES,
|
||||
});
|
||||
|
||||
// Read-your-writes gate: the estimator samples replica lag (reader-side only, paused
|
||||
// when idle) and the router delays wake hydrates by it, anchored to each record's
|
||||
// updatedAtMs — so a publish racing the replica's apply is waited out, not read stale.
|
||||
const lagEstimator =
|
||||
env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_GATE_ENABLED === "1"
|
||||
? new ReplicaLagEstimator({
|
||||
source: createPostgresReplicaLagSource($replica),
|
||||
sampleIntervalMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_SAMPLE_INTERVAL_MS,
|
||||
idleAfterMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_IDLE_AFTER_MS,
|
||||
windowMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_WINDOW_MS,
|
||||
defaultLagMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_DEFAULT_MS,
|
||||
observedFloorTtlMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_OBSERVED_FLOOR_TTL_MS,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
// The notifier wrapped so router activity keeps the lag sampler warm.
|
||||
const notifier = getRunChangeNotifier();
|
||||
const source: EnvChangeSource = lagEstimator
|
||||
? {
|
||||
subscribeToEnv(environmentId, onBatch) {
|
||||
lagEstimator.touch();
|
||||
return notifier.subscribeToEnv(environmentId, (records) => {
|
||||
lagEstimator.touch();
|
||||
onBatch(records);
|
||||
});
|
||||
},
|
||||
}
|
||||
: notifier;
|
||||
|
||||
const router = new EnvChangeRouter({
|
||||
source,
|
||||
hydrator: runReader,
|
||||
onHydrate: (runCount) => routerHydrates.add(runCount),
|
||||
replayWindowMs: env.REALTIME_BACKEND_NATIVE_REPLAY_WINDOW_MS,
|
||||
replayMaxRunsPerEnv: env.REALTIME_BACKEND_NATIVE_REPLAY_MAX_RUNS,
|
||||
unsubscribeLingerMs: env.REALTIME_BACKEND_NATIVE_UNSUBSCRIBE_LINGER_MS,
|
||||
onReplay: (result) => replays.add(1, { result }),
|
||||
onReplayEviction: (reason) => replayEvictions.add(1, { reason }),
|
||||
replicaLag: lagEstimator
|
||||
? {
|
||||
getLagMs: () => lagEstimator.getLagMs(),
|
||||
noteObservedLagMs: (lagMs) => lagEstimator.noteObservedLagMs(lagMs),
|
||||
marginMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_MARGIN_MS,
|
||||
maxDelayMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_MAX_DELAY_MS,
|
||||
staleRetries: env.REALTIME_BACKEND_NATIVE_STALE_HYDRATE_RETRIES,
|
||||
onStaleHydrate: (outcome, runCount) => staleHydrates.add(runCount, { outcome }),
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
const client = new NativeRealtimeClient({
|
||||
runReader,
|
||||
runListResolver: new ClickHouseRunListResolver({
|
||||
getClickhouse: (organizationId) =>
|
||||
clickhouseFactory.getClickhouseForOrganization(organizationId, "realtime"),
|
||||
prisma: $replica,
|
||||
}),
|
||||
router,
|
||||
limiter,
|
||||
cachedLimitProvider: {
|
||||
async getCachedLimit(organizationId, defaultValue) {
|
||||
const result = await getCachedLimit(
|
||||
organizationId,
|
||||
"realtimeConcurrentConnections",
|
||||
defaultValue
|
||||
);
|
||||
return result.val;
|
||||
},
|
||||
},
|
||||
defaultConcurrencyLimit: env.REALTIME_BACKEND_NATIVE_DEFAULT_CONCURRENCY_LIMIT,
|
||||
livePollTimeoutMs: env.REALTIME_BACKEND_NATIVE_LIVE_POLL_TIMEOUT_MS,
|
||||
livePollJitterRatio: env.REALTIME_BACKEND_NATIVE_LIVE_POLL_JITTER_RATIO,
|
||||
maximumCreatedAtFilterAgeMs: env.REALTIME_MAXIMUM_CREATED_AT_FILTER_AGE_IN_MS,
|
||||
maxListResults: env.REALTIME_BACKEND_NATIVE_MAX_LIST_RESULTS,
|
||||
runSetResolveCacheTtlMs: env.REALTIME_BACKEND_NATIVE_RUNSET_CACHE_TTL_MS,
|
||||
runSetResolveCacheMaxEntries: env.REALTIME_BACKEND_NATIVE_RUNSET_CACHE_MAX_ENTRIES,
|
||||
listCacheMaxEntries: env.REALTIME_BACKEND_NATIVE_WORKING_SET_MAX_ENTRIES,
|
||||
workingSetCacheTtlMs: env.REALTIME_BACKEND_NATIVE_WORKING_SET_TTL_MS,
|
||||
runSetCreatedAtBucketMs: env.REALTIME_BACKEND_NATIVE_RUNSET_CREATED_AT_BUCKET_MS,
|
||||
holdOnEmpty: env.REALTIME_BACKEND_NATIVE_HOLD_ON_EMPTY === "1",
|
||||
resolveAdmissionLimit: env.REALTIME_BACKEND_NATIVE_RESOLVE_ADMISSION_LIMIT,
|
||||
replayCursorStore,
|
||||
onWakeup: (reason) => wakeups.add(1, { reason }),
|
||||
onLivePollPath: (path) => livePollPaths.add(1, { path }),
|
||||
onRunSetResolve: (result) => runSetResolves.add(1, { result }),
|
||||
onRunSetQuery: (stage, ms) => runSetQueryMs.record(ms, { stage }),
|
||||
onResolveAdmissionWait: () => resolveAdmissionWaits.add(1),
|
||||
onEmit: (path, lagMs, rowCount) => {
|
||||
deliveryLagMs.record(Math.max(lagMs, 0), { path });
|
||||
emittedRows.record(rowCount);
|
||||
},
|
||||
onBackstopResult: (result) => backstops.add(1, { result }),
|
||||
onConcurrencyRejected: () => concurrencyRejections.add(1),
|
||||
});
|
||||
|
||||
meter
|
||||
.createObservableGauge("realtime_native.working_set_size", {
|
||||
description:
|
||||
"Entries in the per-handle working-set cache (one per active multi-run feed session).",
|
||||
})
|
||||
.addCallback((result) => result.observe(client.workingSetCacheSize));
|
||||
|
||||
meter
|
||||
.createObservableGauge("realtime_native.resolve_admission_in_use", {
|
||||
description:
|
||||
"Fresh ClickHouse resolves currently holding an admission permit (live concurrency against the gate's limit).",
|
||||
})
|
||||
.addCallback((result) => result.observe(client.resolveAdmissionInUse));
|
||||
|
||||
meter
|
||||
.createObservableGauge("realtime_native.held_feeds", {
|
||||
description: "Long-polls currently held, by feed kind — the system's capacity unit.",
|
||||
})
|
||||
.addCallback((result) => {
|
||||
const counts = router.heldFeedCounts;
|
||||
result.observe(counts.run, { kind: "run" });
|
||||
result.observe(counts.tag, { kind: "tag" });
|
||||
result.observe(counts.batch, { kind: "batch" });
|
||||
});
|
||||
|
||||
meter
|
||||
.createObservableGauge("realtime_native.active_envs", {
|
||||
description:
|
||||
"Environments currently routed on this instance (held feeds + lingering subscriptions).",
|
||||
})
|
||||
.addCallback((result) => result.observe(router.activeEnvCount));
|
||||
|
||||
if (lagEstimator) {
|
||||
meter
|
||||
.createObservableGauge("realtime_native.replica_lag_estimate_ms", {
|
||||
description:
|
||||
"The read-your-writes gate's current replica-lag estimate (max sample in the window). Wake hydrates are delayed by roughly this much past each change's commit.",
|
||||
})
|
||||
.addCallback((result) => result.observe(lagEstimator.getLagMs()));
|
||||
}
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
export function getNativeRealtimeClient(): NativeRealtimeClient {
|
||||
return singleton("nativeRealtimeClient", initializeNativeRealtimeClient);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { Callback, Result } from "ioredis";
|
||||
import type { RedisClient, RedisWithClusterOptions } from "~/redis.server";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
import { logger } from "../logger.server";
|
||||
|
||||
export type RealtimeConcurrencyLimiterOptions = {
|
||||
redis: RedisWithClusterOptions;
|
||||
keyPrefix: string;
|
||||
/** How long a tracked request lives before it's swept as stale (seconds). */
|
||||
expiryTimeInSeconds?: number;
|
||||
connectionName?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Per-environment concurrent-connection limiter for realtime long-polls; a standalone copy of the limiter in
|
||||
* `realtimeClient.server.ts` (identical Lua + key shape, different key prefix) so the native backend tracks independently.
|
||||
*/
|
||||
export class RealtimeConcurrencyLimiter {
|
||||
private redis: RedisClient;
|
||||
private expiryTimeInSeconds: number;
|
||||
|
||||
constructor(private options: RealtimeConcurrencyLimiterOptions) {
|
||||
this.redis = createRedisClient(
|
||||
options.connectionName ?? "trigger:realtime:native:concurrency",
|
||||
options.redis
|
||||
);
|
||||
this.expiryTimeInSeconds = options.expiryTimeInSeconds ?? 60 * 5;
|
||||
this.#registerCommands();
|
||||
}
|
||||
|
||||
async incrementAndCheck(
|
||||
environmentId: string,
|
||||
requestId: string,
|
||||
limit: number
|
||||
): Promise<boolean> {
|
||||
const key = this.#getKey(environmentId);
|
||||
const now = Date.now();
|
||||
|
||||
const result = await this.redis.incrementAndCheckRealtimeNativeConcurrency(
|
||||
key,
|
||||
now.toString(),
|
||||
requestId,
|
||||
this.expiryTimeInSeconds.toString(),
|
||||
(now - this.expiryTimeInSeconds * 1000).toString(),
|
||||
limit.toString()
|
||||
);
|
||||
|
||||
return result === 1;
|
||||
}
|
||||
|
||||
async decrement(environmentId: string, requestId: string): Promise<void> {
|
||||
const key = this.#getKey(environmentId);
|
||||
await this.redis.zrem(key, requestId);
|
||||
}
|
||||
|
||||
#getKey(environmentId: string): string {
|
||||
return `${this.options.keyPrefix}:${environmentId}`;
|
||||
}
|
||||
|
||||
#registerCommands() {
|
||||
this.redis.defineCommand("incrementAndCheckRealtimeNativeConcurrency", {
|
||||
numberOfKeys: 1,
|
||||
lua: /* lua */ `
|
||||
local concurrencyKey = KEYS[1]
|
||||
|
||||
local timestamp = tonumber(ARGV[1])
|
||||
local requestId = ARGV[2]
|
||||
local expiryTime = tonumber(ARGV[3])
|
||||
local cutoffTime = tonumber(ARGV[4])
|
||||
local limit = tonumber(ARGV[5])
|
||||
|
||||
-- Remove expired entries
|
||||
redis.call('ZREMRANGEBYSCORE', concurrencyKey, '-inf', cutoffTime)
|
||||
|
||||
-- Add the new request to the sorted set
|
||||
redis.call('ZADD', concurrencyKey, timestamp, requestId)
|
||||
|
||||
-- Set the expiry time on the key
|
||||
redis.call('EXPIRE', concurrencyKey, expiryTime)
|
||||
|
||||
-- Get the total number of concurrent requests
|
||||
local totalRequests = redis.call('ZCARD', concurrencyKey)
|
||||
|
||||
-- Check if the limit has been exceeded
|
||||
if totalRequests > limit then
|
||||
redis.call('ZREM', concurrencyKey, requestId)
|
||||
return 0
|
||||
end
|
||||
|
||||
return 1
|
||||
`,
|
||||
});
|
||||
|
||||
this.redis.on("error", (error) => {
|
||||
logger.error("[realtimeConcurrencyLimiter] redis error", { error });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
declare module "ioredis" {
|
||||
interface RedisCommander<Context> {
|
||||
incrementAndCheckRealtimeNativeConcurrency(
|
||||
key: string,
|
||||
timestamp: string,
|
||||
requestId: string,
|
||||
expiryTime: string,
|
||||
cutoffTime: string,
|
||||
limit: string,
|
||||
callback?: Callback<number>
|
||||
): Result<number, Context>;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
import type { LogLevel } from "@trigger.dev/core/logger";
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
import type { RedisOptions } from "ioredis";
|
||||
import Redis from "ioredis";
|
||||
import { defaultReconnectOnError } from "@internal/redis";
|
||||
import { env } from "~/env.server";
|
||||
import type { StreamIngestor, StreamResponder, StreamResponseOptions } from "./types";
|
||||
|
||||
export type RealtimeStreamsOptions = {
|
||||
redis: RedisOptions | undefined;
|
||||
logger?: Logger;
|
||||
logLevel?: LogLevel;
|
||||
inactivityTimeoutMs?: number; // Close stream after this many ms of no new data (default: 15000)
|
||||
};
|
||||
|
||||
// Legacy constant for backward compatibility (no longer written, but still recognized when reading)
|
||||
const END_SENTINEL = "<<CLOSE_STREAM>>";
|
||||
|
||||
// Internal types for stream pipeline
|
||||
type StreamChunk =
|
||||
| { type: "ping" }
|
||||
| { type: "data"; redisId: string; data: string }
|
||||
| { type: "legacy-data"; redisId: string; data: string };
|
||||
|
||||
// Class implementing both interfaces
|
||||
export class RedisRealtimeStreams implements StreamIngestor, StreamResponder {
|
||||
private logger: Logger;
|
||||
private inactivityTimeoutMs: number;
|
||||
// Shared connection for short-lived non-blocking operations (XADD, XREVRANGE, EXPIRE).
|
||||
// Lazily created on first use so we don't open a connection if only streamResponse is called.
|
||||
private _sharedRedis: Redis | undefined;
|
||||
|
||||
constructor(private options: RealtimeStreamsOptions) {
|
||||
this.logger = options.logger ?? new Logger("RedisRealtimeStreams", options.logLevel ?? "info");
|
||||
this.inactivityTimeoutMs = options.inactivityTimeoutMs ?? 15000; // Default: 15 seconds
|
||||
}
|
||||
|
||||
private get sharedRedis(): Redis {
|
||||
if (!this._sharedRedis) {
|
||||
this._sharedRedis = new Redis({
|
||||
reconnectOnError: defaultReconnectOnError,
|
||||
...this.options.redis,
|
||||
connectionName: "realtime:shared",
|
||||
});
|
||||
}
|
||||
return this._sharedRedis;
|
||||
}
|
||||
|
||||
async initializeStream(
|
||||
runId: string,
|
||||
streamId: string
|
||||
): Promise<{ responseHeaders?: Record<string, string> }> {
|
||||
return {};
|
||||
}
|
||||
|
||||
async streamResponse(
|
||||
request: Request,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
signal: AbortSignal,
|
||||
options?: StreamResponseOptions
|
||||
): Promise<Response> {
|
||||
const redis = new Redis({
|
||||
reconnectOnError: defaultReconnectOnError,
|
||||
...this.options.redis,
|
||||
connectionName: "realtime:streamResponse",
|
||||
});
|
||||
const streamKey = `stream:${runId}:${streamId}`;
|
||||
let isCleanedUp = false;
|
||||
|
||||
const stream = new ReadableStream<StreamChunk>({
|
||||
start: async (controller) => {
|
||||
// Start from lastEventId if provided, otherwise from beginning
|
||||
let lastId = options?.lastEventId ?? "0";
|
||||
let retryCount = 0;
|
||||
const maxRetries = 3;
|
||||
let lastDataTime = Date.now();
|
||||
let lastEnqueueTime = Date.now();
|
||||
const blockTimeMs = 5000;
|
||||
const pingIntervalMs = 10000; // 10 seconds
|
||||
|
||||
if (options?.lastEventId) {
|
||||
this.logger.debug("[RealtimeStreams][streamResponse] Resuming from lastEventId", {
|
||||
streamKey,
|
||||
lastEventId: options?.lastEventId,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
while (!signal.aborted) {
|
||||
// Check if we need to send a ping
|
||||
const timeSinceLastEnqueue = Date.now() - lastEnqueueTime;
|
||||
if (timeSinceLastEnqueue >= pingIntervalMs) {
|
||||
controller.enqueue({ type: "ping" });
|
||||
lastEnqueueTime = Date.now();
|
||||
}
|
||||
|
||||
// Compute inactivity threshold once to use consistently in both branches
|
||||
const inactivityThresholdMs = options?.timeoutInSeconds
|
||||
? options.timeoutInSeconds * 1000
|
||||
: this.inactivityTimeoutMs;
|
||||
|
||||
try {
|
||||
const messages = await redis.xread(
|
||||
"COUNT",
|
||||
100,
|
||||
"BLOCK",
|
||||
blockTimeMs,
|
||||
"STREAMS",
|
||||
streamKey,
|
||||
lastId
|
||||
);
|
||||
|
||||
retryCount = 0;
|
||||
|
||||
if (messages && messages.length > 0) {
|
||||
const [_key, entries] = messages[0];
|
||||
let foundData = false;
|
||||
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const [id, fields] = entries[i];
|
||||
lastId = id;
|
||||
|
||||
if (fields && fields.length >= 2) {
|
||||
// Extract the data field from the Redis entry
|
||||
// Fields format: ["field1", "value1", "field2", "value2", ...]
|
||||
let data: string | null = null;
|
||||
|
||||
for (let j = 0; j < fields.length; j += 2) {
|
||||
if (fields[j] === "data") {
|
||||
data = fields[j + 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle legacy entries that don't have field names (just data at index 1)
|
||||
if (data === null && fields.length >= 2) {
|
||||
data = fields[1];
|
||||
}
|
||||
|
||||
if (data) {
|
||||
// Skip legacy END_SENTINEL entries (backward compatibility)
|
||||
if (data === END_SENTINEL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enqueue structured chunk with Redis stream ID
|
||||
controller.enqueue({
|
||||
type: "data",
|
||||
redisId: id,
|
||||
data,
|
||||
});
|
||||
|
||||
foundData = true;
|
||||
lastDataTime = Date.now();
|
||||
lastEnqueueTime = Date.now();
|
||||
|
||||
if (signal.aborted) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we didn't find any data in this batch, might have only seen sentinels
|
||||
if (!foundData) {
|
||||
// Check for inactivity timeout
|
||||
const inactiveMs = Date.now() - lastDataTime;
|
||||
if (inactiveMs >= inactivityThresholdMs) {
|
||||
this.logger.debug(
|
||||
"[RealtimeStreams][streamResponse] Closing stream due to inactivity",
|
||||
{
|
||||
streamKey,
|
||||
inactiveMs,
|
||||
threshold: inactivityThresholdMs,
|
||||
}
|
||||
);
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No messages received (timed out on BLOCK)
|
||||
// Check for inactivity timeout
|
||||
const inactiveMs = Date.now() - lastDataTime;
|
||||
if (inactiveMs >= inactivityThresholdMs) {
|
||||
this.logger.debug(
|
||||
"[RealtimeStreams][streamResponse] Closing stream due to inactivity",
|
||||
{
|
||||
streamKey,
|
||||
inactiveMs,
|
||||
threshold: inactivityThresholdMs,
|
||||
}
|
||||
);
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal.aborted) break;
|
||||
|
||||
this.logger.error(
|
||||
"[RealtimeStreams][streamResponse] Error reading from Redis stream:",
|
||||
{
|
||||
error,
|
||||
}
|
||||
);
|
||||
retryCount++;
|
||||
if (retryCount >= maxRetries) throw error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error("[RealtimeStreams][streamResponse] Fatal error in stream processing:", {
|
||||
error,
|
||||
});
|
||||
controller.error(error);
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
},
|
||||
cancel: async () => {
|
||||
await cleanup();
|
||||
},
|
||||
})
|
||||
.pipeThrough(
|
||||
// Transform 1: Buffer partial lines across Redis entries
|
||||
(() => {
|
||||
let buffer = "";
|
||||
let lastRedisId = "0";
|
||||
|
||||
return new TransformStream<StreamChunk, StreamChunk & { line: string }>({
|
||||
transform(chunk, controller) {
|
||||
if (chunk.type === "ping") {
|
||||
controller.enqueue(chunk as any);
|
||||
} else if (chunk.type === "data" || chunk.type === "legacy-data") {
|
||||
// Buffer partial lines: accumulate until we see newlines
|
||||
buffer += chunk.data;
|
||||
|
||||
// Split on newlines
|
||||
const lines = buffer.split("\n");
|
||||
|
||||
// The last element might be incomplete, hold it back in buffer
|
||||
buffer = lines.pop() || "";
|
||||
|
||||
// Emit complete lines with the Redis ID of the chunk that completed them
|
||||
for (const line of lines) {
|
||||
if (line.trim().length > 0) {
|
||||
controller.enqueue({
|
||||
...chunk,
|
||||
line,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update last Redis ID for next iteration
|
||||
lastRedisId = chunk.redisId;
|
||||
}
|
||||
},
|
||||
flush(controller) {
|
||||
// On stream end, emit any leftover buffered text
|
||||
if (buffer.trim().length > 0) {
|
||||
controller.enqueue({
|
||||
type: "data",
|
||||
redisId: lastRedisId,
|
||||
data: "",
|
||||
line: buffer.trim(),
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
})()
|
||||
)
|
||||
.pipeThrough(
|
||||
// Transform 2: Format as SSE
|
||||
new TransformStream<StreamChunk & { line?: string }, string>({
|
||||
transform(chunk, controller) {
|
||||
if (chunk.type === "ping") {
|
||||
controller.enqueue(`: ping\n\n`);
|
||||
} else if ((chunk.type === "data" || chunk.type === "legacy-data") && chunk.line) {
|
||||
// Use Redis stream ID as SSE event ID
|
||||
controller.enqueue(`id: ${chunk.redisId}\ndata: ${chunk.line}\n\n`);
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
.pipeThrough(new TextEncoderStream());
|
||||
|
||||
async function cleanup() {
|
||||
if (isCleanedUp) return;
|
||||
isCleanedUp = true;
|
||||
// disconnect() tears down the TCP socket immediately, which causes any
|
||||
// pending XREAD BLOCK to reject right away instead of waiting for the
|
||||
// block timeout to elapse. quit() would queue behind the blocking command.
|
||||
redis.disconnect();
|
||||
}
|
||||
|
||||
signal.addEventListener("abort", cleanup, { once: true });
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async ingestData(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
clientId: string,
|
||||
resumeFromChunk?: number
|
||||
): Promise<Response> {
|
||||
const redis = this.sharedRedis;
|
||||
const streamKey = `stream:${runId}:${streamId}`;
|
||||
const startChunk = resumeFromChunk ?? 0;
|
||||
// Start counting from the resume point, not from 0
|
||||
let currentChunkIndex = startChunk;
|
||||
|
||||
try {
|
||||
const textStream = stream.pipeThrough(new TextDecoderStream());
|
||||
const reader = textStream.getReader();
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
|
||||
if (done || !value) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Write each chunk with its index and clientId
|
||||
this.logger.debug("[RedisRealtimeStreams][ingestData] Writing chunk", {
|
||||
streamKey,
|
||||
runId,
|
||||
clientId,
|
||||
chunkIndex: currentChunkIndex,
|
||||
resumeFromChunk: startChunk,
|
||||
value,
|
||||
});
|
||||
|
||||
await redis.xadd(
|
||||
streamKey,
|
||||
"MAXLEN",
|
||||
"~",
|
||||
String(env.REALTIME_STREAM_MAX_LENGTH),
|
||||
"*",
|
||||
"clientId",
|
||||
clientId,
|
||||
"chunkIndex",
|
||||
currentChunkIndex.toString(),
|
||||
"data",
|
||||
value
|
||||
);
|
||||
|
||||
currentChunkIndex++;
|
||||
}
|
||||
|
||||
// Set TTL for cleanup when stream is done
|
||||
await redis.expire(streamKey, env.REALTIME_STREAM_TTL);
|
||||
|
||||
return new Response(null, { status: 200 });
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
if ("code" in error && error.code === "ECONNRESET") {
|
||||
this.logger.info("[RealtimeStreams][ingestData] Connection reset during ingestData:", {
|
||||
error,
|
||||
});
|
||||
return new Response(null, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.error("[RealtimeStreams][ingestData] Error in ingestData:", { error });
|
||||
|
||||
return new Response(null, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
async appendPart(part: string, partId: string, runId: string, streamId: string): Promise<void> {
|
||||
const redis = this.sharedRedis;
|
||||
const streamKey = `stream:${runId}:${streamId}`;
|
||||
|
||||
await redis.xadd(
|
||||
streamKey,
|
||||
"MAXLEN",
|
||||
"~",
|
||||
String(env.REALTIME_STREAM_MAX_LENGTH),
|
||||
"*",
|
||||
"clientId",
|
||||
"",
|
||||
"chunkIndex",
|
||||
"0",
|
||||
"data",
|
||||
JSON.stringify(part) + "\n"
|
||||
);
|
||||
|
||||
// Set TTL for cleanup when stream is done
|
||||
await redis.expire(streamKey, env.REALTIME_STREAM_TTL);
|
||||
}
|
||||
|
||||
async getLastChunkIndex(runId: string, streamId: string, clientId: string): Promise<number> {
|
||||
const redis = this.sharedRedis;
|
||||
const streamKey = `stream:${runId}:${streamId}`;
|
||||
|
||||
try {
|
||||
// Paginate through the stream from newest to oldest until we find this client's last chunk
|
||||
const batchSize = 100;
|
||||
let lastId = "+"; // Start from newest
|
||||
|
||||
while (true) {
|
||||
const entries = await redis.xrevrange(streamKey, lastId, "-", "COUNT", batchSize);
|
||||
|
||||
if (!entries || entries.length === 0) {
|
||||
// Reached the beginning of the stream, no chunks from this client
|
||||
this.logger.debug(
|
||||
"[RedisRealtimeStreams][getLastChunkIndex] No chunks found for client",
|
||||
{
|
||||
streamKey,
|
||||
clientId,
|
||||
}
|
||||
);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Search through this batch for the client's last chunk
|
||||
for (const [_id, fields] of entries) {
|
||||
let entryClientId: string | null = null;
|
||||
let chunkIndex: number | null = null;
|
||||
let data: string | null = null;
|
||||
|
||||
for (let i = 0; i < fields.length; i += 2) {
|
||||
if (fields[i] === "clientId") {
|
||||
entryClientId = fields[i + 1];
|
||||
}
|
||||
if (fields[i] === "chunkIndex") {
|
||||
chunkIndex = parseInt(fields[i + 1], 10);
|
||||
}
|
||||
if (fields[i] === "data") {
|
||||
data = fields[i + 1];
|
||||
}
|
||||
}
|
||||
|
||||
// Skip legacy END_SENTINEL entries (backward compatibility)
|
||||
if (data === END_SENTINEL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this entry is from our client and has a chunkIndex
|
||||
if (entryClientId === clientId && chunkIndex !== null) {
|
||||
this.logger.debug("[RedisRealtimeStreams][getLastChunkIndex] Found last chunk", {
|
||||
streamKey,
|
||||
clientId,
|
||||
chunkIndex,
|
||||
});
|
||||
return chunkIndex;
|
||||
}
|
||||
}
|
||||
|
||||
// Move to next batch (older entries)
|
||||
// Use the ID of the last entry in this batch as the new cursor
|
||||
lastId = `(${entries[entries.length - 1][0]}`; // Exclusive range with (
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error("[RedisRealtimeStreams][getLastChunkIndex] Error getting last chunk:", {
|
||||
error,
|
||||
streamKey,
|
||||
clientId,
|
||||
});
|
||||
// Return -1 to indicate we don't know what the server has
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
async readRecords(): Promise<never> {
|
||||
throw new Error("readRecords is not implemented for Redis realtime streams");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { createRedisClient, type RedisClient, type RedisWithClusterOptions } from "~/redis.server";
|
||||
import { logger } from "../logger.server";
|
||||
import { BoundedTtlCache } from "./boundedTtlCache";
|
||||
|
||||
/**
|
||||
* Per-connection replay cursors ("when did this connection last receive data"), keyed by the
|
||||
* env-prefixed working-set key. Sharing them fleet-wide makes an instance hop look like a normal
|
||||
* inter-poll gap instead of an unknown one, so hops stop triggering cold resolves and full-window
|
||||
* replays. Values are single timestamps, so the shared store stays cheap.
|
||||
*/
|
||||
export interface ReplayCursorStore {
|
||||
/** The connection's last-response timestamp; undefined on miss OR error (the caller
|
||||
* degrades to a cold probe / full-window replay, never blocks the poll). */
|
||||
get(key: string): Promise<number | undefined>;
|
||||
/** Fire-and-forget stamp; must never throw. */
|
||||
set(key: string, ms: number): void;
|
||||
}
|
||||
|
||||
/** Per-instance fallback with the same shape (used when the shared store is disabled, and in tests). */
|
||||
export class InMemoryReplayCursorStore implements ReplayCursorStore {
|
||||
readonly #cache: BoundedTtlCache<number>;
|
||||
|
||||
constructor(ttlMs: number, maxEntries: number) {
|
||||
this.#cache = new BoundedTtlCache<number>(ttlMs, maxEntries);
|
||||
}
|
||||
|
||||
async get(key: string): Promise<number | undefined> {
|
||||
return this.#cache.get(key);
|
||||
}
|
||||
|
||||
set(key: string, ms: number): void {
|
||||
this.#cache.set(key, ms);
|
||||
}
|
||||
}
|
||||
|
||||
export type RedisReplayCursorStoreOptions = {
|
||||
redis: RedisWithClusterOptions;
|
||||
/** Entry TTL (ms); matches the working-set TTL so both views of a connection age out together. */
|
||||
ttlMs: number;
|
||||
/** Read deadline (ms): a slow or down Redis degrades the poll to a cold probe instead of stalling it. */
|
||||
getTimeoutMs?: number;
|
||||
keyPrefix?: string;
|
||||
connectionName?: string;
|
||||
/** Observability hook: a store op settled (errors are the degradation signal, not failures). */
|
||||
onResult?: (op: "get" | "set", ok: boolean) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_KEY_PREFIX = "realtime:replay-cursor:";
|
||||
const DEFAULT_GET_TIMEOUT_MS = 250;
|
||||
const TIMED_OUT = Symbol("replay-cursor-get-timeout");
|
||||
|
||||
export class RedisReplayCursorStore implements ReplayCursorStore {
|
||||
#client: RedisClient | undefined;
|
||||
|
||||
constructor(private readonly options: RedisReplayCursorStoreOptions) {}
|
||||
|
||||
async get(key: string): Promise<number | undefined> {
|
||||
try {
|
||||
const raw = await this.#getWithDeadline(this.#key(key));
|
||||
if (raw === TIMED_OUT) {
|
||||
this.options.onResult?.("get", false);
|
||||
logger.warn("[replayCursorStore] replay-cursor read timed out", { key });
|
||||
return undefined;
|
||||
}
|
||||
this.options.onResult?.("get", true);
|
||||
if (raw === null) {
|
||||
return undefined;
|
||||
}
|
||||
const ms = Number(raw);
|
||||
return Number.isFinite(ms) && ms > 0 ? ms : undefined;
|
||||
} catch (error) {
|
||||
this.options.onResult?.("get", false);
|
||||
logger.error("[replayCursorStore] failed to read a replay cursor", { error, key });
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** GET raced against the read deadline (ioredis queues commands while disconnected, which
|
||||
* would otherwise stall every poll start through an outage). */
|
||||
#getWithDeadline(key: string): Promise<string | null | typeof TIMED_OUT> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => resolve(TIMED_OUT),
|
||||
this.options.getTimeoutMs ?? DEFAULT_GET_TIMEOUT_MS
|
||||
);
|
||||
timer.unref?.();
|
||||
this.#ensureClient()
|
||||
.get(key)
|
||||
.then(
|
||||
(value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
set(key: string, ms: number): void {
|
||||
try {
|
||||
this.#ensureClient()
|
||||
.set(this.#key(key), String(ms), "PX", this.options.ttlMs)
|
||||
.then(
|
||||
() => this.options.onResult?.("set", true),
|
||||
(error) => {
|
||||
this.options.onResult?.("set", false);
|
||||
logger.error("[replayCursorStore] failed to write a replay cursor", { error, key });
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
this.options.onResult?.("set", false);
|
||||
logger.error("[replayCursorStore] failed to write a replay cursor", { error, key });
|
||||
}
|
||||
}
|
||||
|
||||
async quit(): Promise<void> {
|
||||
const client = this.#client;
|
||||
this.#client = undefined;
|
||||
if (!client) return;
|
||||
try {
|
||||
// Bounded graceful QUIT; cursor writes are best-effort, so force-close beyond it.
|
||||
await Promise.race([client.quit(), new Promise((resolve) => setTimeout(resolve, 500))]);
|
||||
} catch {
|
||||
// force-close below
|
||||
}
|
||||
client.disconnect();
|
||||
}
|
||||
|
||||
#key(key: string): string {
|
||||
return `${this.options.keyPrefix ?? DEFAULT_KEY_PREFIX}${key}`;
|
||||
}
|
||||
|
||||
#ensureClient(): RedisClient {
|
||||
if (!this.#client) {
|
||||
this.#client = createRedisClient(
|
||||
this.options.connectionName ?? "trigger:realtime:replay-cursors",
|
||||
this.options.redis
|
||||
);
|
||||
}
|
||||
return this.#client;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
/**
|
||||
* ReplicaLagEstimator — tracks how far the read replica trails the primary so the
|
||||
* EnvChangeRouter can delay wake-path hydrates just long enough to read their own writes.
|
||||
* Two inputs: a ReplicaLagSource (active, reader-side only — never queries the primary)
|
||||
* sampled on an interval while the router is busy, and passive observations fed back by
|
||||
* the router's stale-hydrate tripwire. The estimate is the max over a short window —
|
||||
* floored by recent observations — so spikes widen the delay immediately and decay back
|
||||
* out as fresh samples land.
|
||||
*/
|
||||
|
||||
type RawQueryable = {
|
||||
$queryRawUnsafe<T = unknown>(query: string): Promise<T>;
|
||||
};
|
||||
|
||||
/** A dialect-specific reader-side lag measure. `sampleLagMs()` returns the current lag in
|
||||
* ms, or undefined when lag is genuinely unmeasurable right now (NOT an error — errors
|
||||
* throw, and the composing source uses them to rule the dialect out). */
|
||||
export interface ReplicaLagSource {
|
||||
readonly name: string;
|
||||
sampleLagMs(): Promise<number | undefined>;
|
||||
}
|
||||
|
||||
/** Aurora: replicas share the storage layer and reject every standard WAL function;
|
||||
* `aurora_replica_status()` is the only live lag source. Max across readers, since the
|
||||
* `$replica` pool balances over all of them. No reader rows = `$replica` is the writer =
|
||||
* no lag. Throws on non-Aurora (the function doesn't exist). */
|
||||
export class AuroraReplicaLagSource implements ReplicaLagSource {
|
||||
readonly name = "aurora";
|
||||
|
||||
constructor(private readonly db: RawQueryable) {}
|
||||
|
||||
async sampleLagMs(): Promise<number | undefined> {
|
||||
const rows = await this.db.$queryRawUnsafe<{ lag: number | null }[]>(
|
||||
`SELECT max(replica_lag_in_msec)::float8 AS lag FROM aurora_replica_status() WHERE session_id <> 'MASTER_SESSION_ID' AND replica_lag_in_msec IS NOT NULL`
|
||||
);
|
||||
const lag = rows[0]?.lag;
|
||||
return typeof lag === "number" && Number.isFinite(lag) ? Math.max(0, lag) : 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Vanilla PG streaming replication. A primary (not in recovery — no replica configured,
|
||||
* `$replica` is the writer) has no lag by definition; a caught-up replica (receive LSN ==
|
||||
* replay LSN) reports 0. Mid-apply there is NO honest reader-side timestamp measure —
|
||||
* `now() - pg_last_xact_replay_timestamp()` reads as the full inter-write gap on
|
||||
* low-traffic systems, which (measured locally) pins the estimate at the delay cap — so
|
||||
* mid-apply reports undefined and the tripwire's observed-staleness floor carries the
|
||||
* estimate instead. */
|
||||
export class VanillaPgReplicaLagSource implements ReplicaLagSource {
|
||||
readonly name = "vanilla-pg";
|
||||
|
||||
constructor(private readonly db: RawQueryable) {}
|
||||
|
||||
async sampleLagMs(): Promise<number | undefined> {
|
||||
const rows = await this.db.$queryRawUnsafe<{ caught_up: boolean | null }[]>(
|
||||
`SELECT CASE
|
||||
WHEN NOT pg_is_in_recovery() THEN true
|
||||
WHEN pg_last_wal_receive_lsn() IS NOT DISTINCT FROM pg_last_wal_replay_lsn() THEN true
|
||||
ELSE false
|
||||
END AS caught_up`
|
||||
);
|
||||
return rows[0]?.caught_up ? 0 : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Composes dialect sources: the first whose sample succeeds is selected and used from
|
||||
* then on; a database where none work degrades to never-measuring (the estimator then
|
||||
* runs on its default + tripwire observations). Selection is by thrown-vs-returned —
|
||||
* sources throw on unsupported dialects and return undefined for "can't measure now". */
|
||||
export class FirstSupportedReplicaLagSource implements ReplicaLagSource {
|
||||
/** undefined = not probed yet; null = no candidate works here. */
|
||||
#selected: ReplicaLagSource | null | undefined;
|
||||
|
||||
constructor(private readonly candidates: ReplicaLagSource[]) {}
|
||||
|
||||
get name(): string {
|
||||
return this.#selected ? this.#selected.name : "undetected";
|
||||
}
|
||||
|
||||
async sampleLagMs(): Promise<number | undefined> {
|
||||
if (this.#selected === null) {
|
||||
return undefined;
|
||||
}
|
||||
if (this.#selected) {
|
||||
// Transient errors don't unselect the dialect; the sample is just skipped.
|
||||
try {
|
||||
return await this.#selected.sampleLagMs();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
for (const candidate of this.candidates) {
|
||||
try {
|
||||
const lag = await candidate.sampleLagMs();
|
||||
this.#selected = candidate;
|
||||
logger.info("[replicaLagEstimator] selected lag source", { source: candidate.name });
|
||||
return lag;
|
||||
} catch {
|
||||
// unsupported dialect; try the next
|
||||
}
|
||||
}
|
||||
this.#selected = null;
|
||||
logger.warn(
|
||||
"[replicaLagEstimator] no usable lag source; relying on default + tripwire observations"
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The standard composition for a Prisma replica client. */
|
||||
export function createPostgresReplicaLagSource(replica: RawQueryable): ReplicaLagSource {
|
||||
return new FirstSupportedReplicaLagSource([
|
||||
new AuroraReplicaLagSource(replica),
|
||||
new VanillaPgReplicaLagSource(replica),
|
||||
]);
|
||||
}
|
||||
|
||||
export type ReplicaLagEstimatorOptions = {
|
||||
source: ReplicaLagSource;
|
||||
/** Sample cadence while active. */
|
||||
sampleIntervalMs?: number;
|
||||
/** Stop sampling this long after the last touch(); the next touch resumes. */
|
||||
idleAfterMs?: number;
|
||||
/** The estimate is the max sample inside this window. */
|
||||
windowMs?: number;
|
||||
/** Estimate before any sample lands (and the floor when sampling is unavailable). */
|
||||
defaultLagMs?: number;
|
||||
/** Ceiling on accepted samples — shields the estimate from a wild observation. */
|
||||
maxLagMs?: number;
|
||||
/** How long a tripwire observation floors the estimate. Sources that can't measure
|
||||
* mid-apply lag (vanilla PG) return nothing, so without this floor the estimate decays
|
||||
* to the caught-up zeros within windowMs and every ~window one wake pays a stale retry
|
||||
* to re-learn. */
|
||||
observedFloorTtlMs?: number;
|
||||
/** Observability: a sample (active or passive) was accepted. */
|
||||
onSample?: (lagMs: number, source: "probe" | "observed") => void;
|
||||
};
|
||||
|
||||
const DEFAULT_SAMPLE_INTERVAL_MS = 250;
|
||||
const DEFAULT_IDLE_AFTER_MS = 30_000;
|
||||
const DEFAULT_WINDOW_MS = 5_000;
|
||||
const DEFAULT_DEFAULT_LAG_MS = 30;
|
||||
const DEFAULT_MAX_LAG_MS = 60_000;
|
||||
const DEFAULT_OBSERVED_FLOOR_TTL_MS = 60_000;
|
||||
|
||||
export class ReplicaLagEstimator {
|
||||
readonly #sampleIntervalMs: number;
|
||||
readonly #idleAfterMs: number;
|
||||
readonly #windowMs: number;
|
||||
readonly #defaultLagMs: number;
|
||||
readonly #maxLagMs: number;
|
||||
readonly #observedFloorTtlMs: number;
|
||||
#samples: { atMs: number; lagMs: number }[] = [];
|
||||
#lastKnownLagMs: number | undefined;
|
||||
#observedFloorLagMs = 0;
|
||||
#observedFloorAtMs = 0;
|
||||
#lastTouchMs = 0;
|
||||
#timer: ReturnType<typeof setInterval> | undefined;
|
||||
#sampling = false;
|
||||
|
||||
constructor(private readonly options: ReplicaLagEstimatorOptions) {
|
||||
this.#sampleIntervalMs = options.sampleIntervalMs ?? DEFAULT_SAMPLE_INTERVAL_MS;
|
||||
this.#idleAfterMs = options.idleAfterMs ?? DEFAULT_IDLE_AFTER_MS;
|
||||
this.#windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
|
||||
this.#defaultLagMs = options.defaultLagMs ?? DEFAULT_DEFAULT_LAG_MS;
|
||||
this.#maxLagMs = options.maxLagMs ?? DEFAULT_MAX_LAG_MS;
|
||||
this.#observedFloorTtlMs = options.observedFloorTtlMs ?? DEFAULT_OBSERVED_FLOOR_TTL_MS;
|
||||
}
|
||||
|
||||
/** Mark router activity; starts (or keeps) the sampler running. */
|
||||
touch() {
|
||||
this.#lastTouchMs = Date.now();
|
||||
if (!this.#timer) {
|
||||
this.#timer = setInterval(() => this.#tick(), this.#sampleIntervalMs);
|
||||
this.#timer.unref?.();
|
||||
// Sample immediately so the first wake after idle doesn't run on a stale estimate.
|
||||
this.#tick();
|
||||
}
|
||||
}
|
||||
|
||||
/** Current lag estimate (ms): the max recent sample (else last known, else the default),
|
||||
* floored by the latest tripwire observation while it's fresh. Never throws. */
|
||||
getLagMs(): number {
|
||||
const now = Date.now();
|
||||
const cutoff = now - this.#windowMs;
|
||||
let max: number | undefined;
|
||||
for (const sample of this.#samples) {
|
||||
if (sample.atMs >= cutoff && (max === undefined || sample.lagMs > max)) {
|
||||
max = sample.lagMs;
|
||||
}
|
||||
}
|
||||
const base = max ?? this.#lastKnownLagMs ?? this.#defaultLagMs;
|
||||
const floor =
|
||||
now - this.#observedFloorAtMs < this.#observedFloorTtlMs ? this.#observedFloorLagMs : 0;
|
||||
return Math.max(base, floor);
|
||||
}
|
||||
|
||||
/** Feedback from the stale-hydrate tripwire: a read provably ran at least this far
|
||||
* behind the primary. Widens the estimate immediately AND floors it for a while —
|
||||
* sources that can't measure mid-apply lag would otherwise decay it straight back. */
|
||||
noteObservedLagMs(lagMs: number) {
|
||||
const clamped = Math.min(Math.max(0, lagMs), this.#maxLagMs);
|
||||
const floorExpired = Date.now() - this.#observedFloorAtMs >= this.#observedFloorTtlMs;
|
||||
if (clamped >= this.#observedFloorLagMs || floorExpired) {
|
||||
this.#observedFloorLagMs = clamped;
|
||||
this.#observedFloorAtMs = Date.now();
|
||||
}
|
||||
this.#accept(lagMs, "observed");
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (this.#timer) {
|
||||
clearInterval(this.#timer);
|
||||
this.#timer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
#accept(lagMs: number, source: "probe" | "observed") {
|
||||
if (!Number.isFinite(lagMs)) {
|
||||
return;
|
||||
}
|
||||
const clamped = Math.min(Math.max(0, lagMs), this.#maxLagMs);
|
||||
const now = Date.now();
|
||||
this.#samples.push({ atMs: now, lagMs: clamped });
|
||||
this.#lastKnownLagMs = clamped;
|
||||
const cutoff = now - this.#windowMs;
|
||||
while (this.#samples.length > 0 && this.#samples[0].atMs < cutoff) {
|
||||
this.#samples.shift();
|
||||
}
|
||||
this.options.onSample?.(clamped, source);
|
||||
}
|
||||
|
||||
#tick() {
|
||||
if (Date.now() - this.#lastTouchMs > this.#idleAfterMs) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
if (this.#sampling) {
|
||||
return; // a slow sample shouldn't stack
|
||||
}
|
||||
this.#sampling = true;
|
||||
this.options.source
|
||||
.sampleLagMs()
|
||||
.then((lagMs) => {
|
||||
if (lagMs !== undefined) {
|
||||
this.#accept(lagMs, "probe");
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
// sampling errors never propagate; the estimate just ages
|
||||
})
|
||||
.finally(() => {
|
||||
this.#sampling = false;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { $replica } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { FEATURE_FLAG } from "~/v3/featureFlags";
|
||||
import { makeFlag } from "~/v3/featureFlags.server";
|
||||
import { logger } from "../logger.server";
|
||||
import { type RealtimeEnvironment } from "../realtimeClient.server";
|
||||
import { realtimeClient } from "../realtimeClientGlobal.server";
|
||||
import { BoundedTtlCache } from "./boundedTtlCache";
|
||||
import { type RealtimeStreamClient } from "./nativeRealtimeClient.server";
|
||||
import { getNativeRealtimeClient } from "./nativeRealtimeClientInstance.server";
|
||||
import { getShadowRealtimeClient } from "./shadowRealtimeClientInstance.server";
|
||||
|
||||
type RealtimeBackend = "electric" | "native" | "shadow";
|
||||
|
||||
// Two gates: the env master switch, then the per-org `realtimeBackend` feature flag (cached so
|
||||
// long-polls don't hit the DB per request), falling back to REALTIME_BACKEND_DEFAULT.
|
||||
const nativeBackendEnabled = env.REALTIME_BACKEND_NATIVE_ENABLED === "1";
|
||||
|
||||
const flag = singleton("realtimeBackendFlag", () => makeFlag($replica));
|
||||
const backendCache = singleton(
|
||||
"realtimeBackendCache",
|
||||
() =>
|
||||
new BoundedTtlCache<RealtimeBackend>(
|
||||
env.REALTIME_BACKEND_FLAG_CACHE_TTL_MS,
|
||||
env.REALTIME_BACKEND_FLAG_CACHE_MAX_ENTRIES
|
||||
)
|
||||
);
|
||||
|
||||
export async function resolveRealtimeStreamClient(
|
||||
environment: RealtimeEnvironment & { organization?: { featureFlags?: unknown } }
|
||||
): Promise<RealtimeStreamClient> {
|
||||
if (!nativeBackendEnabled) {
|
||||
return realtimeClient;
|
||||
}
|
||||
|
||||
// The authenticated environment already carries the org's feature flags; pass them
|
||||
// through so a cache miss doesn't need an extra organization read.
|
||||
const orgFeatureFlags = environment.organization
|
||||
? (environment.organization.featureFlags ?? {})
|
||||
: undefined;
|
||||
|
||||
switch (await getRealtimeBackend(environment.organizationId, orgFeatureFlags)) {
|
||||
case "native":
|
||||
return getNativeRealtimeClient();
|
||||
case "shadow":
|
||||
// The client is still served Electric; the native path is diffed in the background.
|
||||
return getShadowRealtimeClient();
|
||||
case "electric":
|
||||
default:
|
||||
return realtimeClient;
|
||||
}
|
||||
}
|
||||
|
||||
async function getRealtimeBackend(
|
||||
organizationId: string,
|
||||
orgFeatureFlags: unknown | undefined
|
||||
): Promise<RealtimeBackend> {
|
||||
const cached = backendCache.get(organizationId);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
let backend: RealtimeBackend = env.REALTIME_BACKEND_DEFAULT;
|
||||
|
||||
try {
|
||||
const overrides =
|
||||
orgFeatureFlags !== undefined
|
||||
? orgFeatureFlags
|
||||
: (
|
||||
await $replica.organization.findFirst({
|
||||
where: { id: organizationId },
|
||||
select: { featureFlags: true },
|
||||
})
|
||||
)?.featureFlags;
|
||||
|
||||
backend = await flag({
|
||||
key: FEATURE_FLAG.realtimeBackend,
|
||||
defaultValue: env.REALTIME_BACKEND_DEFAULT,
|
||||
overrides: (overrides as Record<string, unknown>) ?? {},
|
||||
});
|
||||
} catch (error) {
|
||||
// Never let a flag lookup failure break the realtime feed.
|
||||
logger.error("[resolveRealtimeStreamClient] failed to resolve realtimeBackend flag", {
|
||||
organizationId,
|
||||
error,
|
||||
});
|
||||
backend = env.REALTIME_BACKEND_DEFAULT;
|
||||
}
|
||||
|
||||
backendCache.set(organizationId, backend);
|
||||
return backend;
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
import type { RedisClient, RedisWithClusterOptions } from "~/redis.server";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
import { logger } from "../logger.server";
|
||||
|
||||
export const CHANGE_RECORD_VERSION = 1;
|
||||
|
||||
/**
|
||||
* A self-describing run-change fact published once to the run's environment channel; row state is
|
||||
* never on the wire. `tags` present (even `[]`) marks a "full" record a feed can classify locally;
|
||||
* `tags` absent marks a "partial" record (envId+runId only) a tag feed must hydrate to classify.
|
||||
*/
|
||||
export type ChangeRecord = {
|
||||
v: number;
|
||||
runId: string;
|
||||
envId: string;
|
||||
tags?: string[];
|
||||
batchId?: string | null;
|
||||
createdAtMs?: number;
|
||||
updatedAtMs?: number;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
/** What a publish site provides; the notifier stamps the version. */
|
||||
export type ChangeRecordInput = Omit<ChangeRecord, "v">;
|
||||
|
||||
export function encodeChangeRecord(record: ChangeRecord): string {
|
||||
return JSON.stringify(record);
|
||||
}
|
||||
|
||||
/** Decode a wire message into a ChangeRecord; a bare/malformed frame degrades to a partial record rather than throwing. */
|
||||
export function decodeChangeRecord(message: string): ChangeRecord {
|
||||
if (message.length === 0 || message[0] !== "{") {
|
||||
return { v: 0, runId: message, envId: "" };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(message) as Partial<ChangeRecord>;
|
||||
if (parsed && typeof parsed.runId === "string") {
|
||||
return {
|
||||
v: parsed.v ?? 0,
|
||||
runId: parsed.runId,
|
||||
envId: parsed.envId ?? "",
|
||||
tags: parsed.tags,
|
||||
batchId: parsed.batchId,
|
||||
createdAtMs: parsed.createdAtMs,
|
||||
updatedAtMs: parsed.updatedAtMs,
|
||||
status: parsed.status,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// fall through to the bare-runId fallback
|
||||
}
|
||||
return { v: 0, runId: message, envId: "" };
|
||||
}
|
||||
|
||||
export type RunChangeNotifierOptions = {
|
||||
redis: RedisWithClusterOptions;
|
||||
/** Channel name prefix; the envId is appended inside a hash-tag for slot locality. */
|
||||
channelPrefix?: string;
|
||||
connectionName?: string;
|
||||
/** Leading-edge throttle (ms) for the per-env channel, bounding the wake rate per env. Defaults to 100ms; 0 disables. */
|
||||
envWakeCoalesceWindowMs?: number;
|
||||
/** Use Redis sharded pub/sub (SSUBSCRIBE/SPUBLISH); cluster-only and requires `clusterOptions.shardedSubscribers`. Defaults to false (classic). */
|
||||
shardedPubSub?: boolean;
|
||||
/** Observability hook: a publish settled (ok) or failed (the leading degradation signal). */
|
||||
onPublishResult?: (ok: boolean) => void;
|
||||
/** Observability hook: a raw channel message arrived (pre-coalesce). */
|
||||
onMessageReceived?: () => void;
|
||||
/** Observability hook: a coalesced batch was delivered to listeners (records per batch). */
|
||||
onBatchDelivered?: (recordCount: number) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_CHANNEL_PREFIX = "realtime:";
|
||||
const DEFAULT_ENV_WAKE_COALESCE_WINDOW_MS = 100;
|
||||
|
||||
/**
|
||||
* RunChangeNotifier — carries "run X changed" facts from write sites to the realtime feeds over ONE
|
||||
* per-environment channel (`<prefix>env:{<envId>}`, hash-tagged so an env stays on one cluster slot).
|
||||
* Uses one shared multiplexed subscriber per process (refcounted), created lazily, and a fire-and-forget
|
||||
* `publish` that never throws — a dropped publish only costs latency because the consumer has a backstop.
|
||||
*/
|
||||
export class RunChangeNotifier {
|
||||
#publisher: RedisClient | undefined;
|
||||
#subscriber: RedisClient | undefined;
|
||||
readonly #listeners = new Map<string, Set<(records: ChangeRecord[]) => void>>();
|
||||
/** Per-channel accumulator of records since the last delivery, deduped by runId (latest per run wins), so a coalesced wake carries every run that moved. */
|
||||
readonly #pending = new Map<string, Map<string, ChangeRecord>>();
|
||||
readonly #channelPrefix: string;
|
||||
readonly #connectionName: string;
|
||||
readonly #coalesceWindowMs: number;
|
||||
/** When true, use sharded pub/sub (SSUBSCRIBE/SPUBLISH/smessage) — see options. */
|
||||
readonly #sharded: boolean;
|
||||
/** Active coalescing windows per channel. */
|
||||
readonly #coalesceTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
/** Channels that received a message while their window was open (need a trailing wake). */
|
||||
readonly #coalesceDirty = new Set<string>();
|
||||
|
||||
constructor(private readonly options: RunChangeNotifierOptions) {
|
||||
this.#channelPrefix = options.channelPrefix ?? DEFAULT_CHANNEL_PREFIX;
|
||||
this.#connectionName = options.connectionName ?? "trigger:realtime:run-change-notifier";
|
||||
this.#coalesceWindowMs = options.envWakeCoalesceWindowMs ?? DEFAULT_ENV_WAKE_COALESCE_WINDOW_MS;
|
||||
this.#sharded = options.shardedPubSub ?? false;
|
||||
}
|
||||
|
||||
/** Fire-and-forget publish of a run-changed fact to the run's environment channel; never throws. */
|
||||
publish(input: ChangeRecordInput): void {
|
||||
const record: ChangeRecord = { v: CHANGE_RECORD_VERSION, ...input };
|
||||
this.#publishToChannel(this.#channelForEnv(record.envId), encodeChangeRecord(record));
|
||||
}
|
||||
|
||||
/** Fire-and-forget publish of many run-changed facts. Never throws. */
|
||||
publishMany(inputs: ChangeRecordInput[]): void {
|
||||
for (const input of inputs) {
|
||||
this.publish(input);
|
||||
}
|
||||
}
|
||||
|
||||
#publishToChannel(channel: string, payload: string): void {
|
||||
try {
|
||||
const publisher = this.#ensurePublisher();
|
||||
// Sharded pub/sub (SPUBLISH) routes to the channel's slot owner; classic PUBLISH
|
||||
// broadcasts cluster-wide. The channel is hash-tagged by envId.
|
||||
const result = this.#sharded
|
||||
? publisher.spublish(channel, payload)
|
||||
: publisher.publish(channel, payload);
|
||||
if (typeof (result as Promise<number>)?.then === "function") {
|
||||
(result as Promise<number>).then(
|
||||
() => this.options.onPublishResult?.(true),
|
||||
(error) => {
|
||||
this.options.onPublishResult?.(false);
|
||||
logger.error("[runChangeNotifier] Failed to publish run-changed notification", {
|
||||
error,
|
||||
channel,
|
||||
});
|
||||
}
|
||||
);
|
||||
} else {
|
||||
this.options.onPublishResult?.(true);
|
||||
}
|
||||
} catch (error) {
|
||||
this.options.onPublishResult?.(false);
|
||||
logger.error("[runChangeNotifier] Failed to publish run-changed notification", {
|
||||
error,
|
||||
channel,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to an env's run-change stream; refcounted over the shared subscriber (first listener SUBSCRIBEs, last UNSUBSCRIBEs). */
|
||||
subscribeToEnv(environmentId: string, onBatch: (records: ChangeRecord[]) => void): () => void {
|
||||
const channel = this.#channelForEnv(environmentId);
|
||||
const subscriber = this.#ensureSubscriber();
|
||||
|
||||
let listeners = this.#listeners.get(channel);
|
||||
if (!listeners) {
|
||||
listeners = new Set();
|
||||
this.#listeners.set(channel, listeners);
|
||||
this.#subscribeChannel(subscriber, channel).catch((error) => {
|
||||
logger.error("[runChangeNotifier] Failed to subscribe to run-change channel", {
|
||||
error,
|
||||
channel,
|
||||
});
|
||||
});
|
||||
}
|
||||
listeners.add(onBatch);
|
||||
|
||||
let unsubscribed = false;
|
||||
return () => {
|
||||
if (unsubscribed) {
|
||||
return;
|
||||
}
|
||||
unsubscribed = true;
|
||||
|
||||
const current = this.#listeners.get(channel);
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
current.delete(onBatch);
|
||||
if (current.size === 0) {
|
||||
// Drop the channel from the map only after Redis confirms UNSUBSCRIBE and no new listener re-subscribed in the meantime.
|
||||
this.#unsubscribeChannel(subscriber, channel)
|
||||
.then(() => {
|
||||
const latest = this.#listeners.get(channel);
|
||||
if (!latest) {
|
||||
return;
|
||||
}
|
||||
if (latest.size === 0) {
|
||||
this.#listeners.delete(channel);
|
||||
} else {
|
||||
// A listener arrived during the in-flight UNSUBSCRIBE; re-subscribe so it keeps receiving (the backstop covers the gap).
|
||||
this.#subscribeChannel(subscriber, channel).catch((error) => {
|
||||
logger.error("[runChangeNotifier] Failed to re-subscribe to run-change channel", {
|
||||
error,
|
||||
channel,
|
||||
});
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
// UNSUBSCRIBE failed (likely still subscribed in Redis): keep the empty map entry so a future subscriber reuses it.
|
||||
logger.error("[runChangeNotifier] Failed to unsubscribe from run-change channel", {
|
||||
error,
|
||||
channel,
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Number of distinct env channels currently subscribed (for metrics). */
|
||||
get activeSubscriptionCount(): number {
|
||||
return this.#listeners.size;
|
||||
}
|
||||
|
||||
async quit(): Promise<void> {
|
||||
for (const timer of this.#coalesceTimers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.#coalesceTimers.clear();
|
||||
this.#coalesceDirty.clear();
|
||||
this.#pending.clear();
|
||||
await Promise.allSettled([this.#subscriber?.quit(), this.#publisher?.quit()]);
|
||||
this.#subscriber = undefined;
|
||||
this.#publisher = undefined;
|
||||
this.#listeners.clear();
|
||||
}
|
||||
|
||||
#ensurePublisher(): RedisClient {
|
||||
if (!this.#publisher) {
|
||||
// Publishes are fire-and-forget with a consumer-side backstop, so a dropped publish is
|
||||
// latency-only. Cap retries (vs ioredis's default 20) so a pub/sub outage rejects publishes
|
||||
// after ~1 reconnect cycle instead of buffering them in memory across the fleet.
|
||||
this.#publisher = createRedisClient(`${this.#connectionName}:pub`, {
|
||||
...this.options.redis,
|
||||
maxRetriesPerRequest: 1,
|
||||
});
|
||||
}
|
||||
return this.#publisher;
|
||||
}
|
||||
|
||||
#ensureSubscriber(): RedisClient {
|
||||
if (!this.#subscriber) {
|
||||
const subscriber = createRedisClient(`${this.#connectionName}:sub`, this.options.redis);
|
||||
const onMessage = (channel: string, message: string) => this.#onMessage(channel, message);
|
||||
// Classic pub/sub delivers "message"; sharded pub/sub delivers "smessage". Register
|
||||
// both so the delivery path is identical regardless of mode.
|
||||
subscriber.on("message", onMessage);
|
||||
subscriber.on("smessage", onMessage);
|
||||
this.#subscriber = subscriber;
|
||||
}
|
||||
return this.#subscriber;
|
||||
}
|
||||
|
||||
/** SUBSCRIBE (classic) vs SSUBSCRIBE (sharded, cluster-only). */
|
||||
#subscribeChannel(subscriber: RedisClient, channel: string): Promise<unknown> {
|
||||
return this.#sharded ? subscriber.ssubscribe(channel) : subscriber.subscribe(channel);
|
||||
}
|
||||
|
||||
/** UNSUBSCRIBE (classic) vs SUNSUBSCRIBE (sharded, cluster-only). */
|
||||
#unsubscribeChannel(subscriber: RedisClient, channel: string): Promise<unknown> {
|
||||
return this.#sharded ? subscriber.sunsubscribe(channel) : subscriber.unsubscribe(channel);
|
||||
}
|
||||
|
||||
#onMessage(channel: string, message: string) {
|
||||
this.options.onMessageReceived?.();
|
||||
// Accumulate the decoded record (deduped by runId) before delivering, so a coalesced
|
||||
// wake carries every run that moved during the window.
|
||||
this.#addPending(channel, decodeChangeRecord(message));
|
||||
|
||||
if (this.#coalesceWindowMs > 0) {
|
||||
this.#deliverCoalesced(channel);
|
||||
return;
|
||||
}
|
||||
this.#deliver(channel);
|
||||
}
|
||||
|
||||
/** Accumulate a record into the channel's pending batch, deduped by runId (a later
|
||||
* record for the same run replaces the earlier one, keeping the freshest keys). */
|
||||
#addPending(channel: string, record: ChangeRecord) {
|
||||
let batch = this.#pending.get(channel);
|
||||
if (!batch) {
|
||||
batch = new Map();
|
||||
this.#pending.set(channel, batch);
|
||||
}
|
||||
batch.set(record.runId, record);
|
||||
}
|
||||
|
||||
#deliver(channel: string) {
|
||||
// Drain the accumulated batch (and clear it) so listeners woken now get every run that
|
||||
// changed since the last delivery, and a later message starts a fresh batch.
|
||||
const batchMap = this.#pending.get(channel);
|
||||
const batch = batchMap ? [...batchMap.values()] : [];
|
||||
this.#pending.delete(channel);
|
||||
|
||||
const listeners = this.#listeners.get(channel);
|
||||
if (!listeners || batch.length === 0) {
|
||||
return;
|
||||
}
|
||||
this.options.onBatchDelivered?.(batch.length);
|
||||
for (const onBatch of [...listeners]) {
|
||||
onBatch(batch);
|
||||
}
|
||||
}
|
||||
|
||||
/** Leading-edge throttle capping the wake rate to ~1/window: deliver the first wake immediately, then one trailing wake per window while activity continues. Lossless. */
|
||||
#deliverCoalesced(channel: string) {
|
||||
if (this.#coalesceTimers.has(channel)) {
|
||||
this.#coalesceDirty.add(channel);
|
||||
return;
|
||||
}
|
||||
this.#deliver(channel);
|
||||
this.#openCoalesceWindow(channel);
|
||||
}
|
||||
|
||||
#openCoalesceWindow(channel: string) {
|
||||
const timer = setTimeout(() => {
|
||||
this.#coalesceTimers.delete(channel);
|
||||
if (this.#coalesceDirty.delete(channel)) {
|
||||
this.#deliver(channel);
|
||||
this.#openCoalesceWindow(channel);
|
||||
}
|
||||
}, this.#coalesceWindowMs);
|
||||
// Don't let a pending coalescing window hold the process open at shutdown.
|
||||
timer.unref?.();
|
||||
this.#coalesceTimers.set(channel, timer);
|
||||
}
|
||||
|
||||
// Hash-tagged (`...{<envId>}`) so all of an env's traffic maps to one cluster slot (one
|
||||
// shard) under sharded pub/sub.
|
||||
#channelForEnv(environmentId: string): string {
|
||||
return `${this.#channelPrefix}env:{${environmentId}}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { env } from "~/env.server";
|
||||
import { engine } from "~/v3/runEngine.server";
|
||||
import { logger } from "../logger.server";
|
||||
import { publishChangeRecord } from "./runChangeNotifierInstance.server";
|
||||
|
||||
/**
|
||||
* Builds and publishes a self-describing `ChangeRecord` for the lifecycle events whose engine-bus payload
|
||||
* already carries env + tags + batchId. Terminal transitions, runAttemptFailed, and runMetadataUpdated publish
|
||||
* from `runEngineHandlers.server.ts` instead. Coverage isn't exhaustive — a dropped transition only adds latency
|
||||
* because the consumer has a periodic backstop full-resolve. The env master switch is `REALTIME_BACKEND_NATIVE_ENABLED`.
|
||||
*/
|
||||
export function registerRunChangeNotifierHandlers() {
|
||||
// Return truthy in every path so singleton() caches this factory and never re-runs it (re-running would attach duplicate engine-bus listeners on dev reload).
|
||||
if (env.REALTIME_BACKEND_NATIVE_ENABLED !== "1") {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Run created: the first signal for a brand-new run (born QUEUED with no status transition), so it surfaces before ClickHouse ingests it.
|
||||
engine.eventBus.on("runCreated", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
|
||||
// Status transitions (checkpoint suspend/resume, pending version, dequeue).
|
||||
engine.eventBus.on("runStatusChanged", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
|
||||
// Dequeue/lock (sets startedAt) and attempt start (DEQUEUED -> EXECUTING) — the
|
||||
// most-watched "my run started" transitions.
|
||||
engine.eventBus.on("runLocked", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
engine.eventBus.on("runAttemptStarted", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
|
||||
engine.eventBus.on("runRetryScheduled", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
|
||||
// Delay lifecycle (delayUntil / queued-after-delay changes).
|
||||
engine.eventBus.on("runDelayRescheduled", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
engine.eventBus.on("runEnqueuedAfterDelay", ({ run, environment }) => {
|
||||
publishChangeRecord({
|
||||
runId: run.id,
|
||||
envId: environment.id,
|
||||
tags: run.runTags,
|
||||
batchId: run.batchId,
|
||||
});
|
||||
});
|
||||
|
||||
logger.info("[runChangeNotifier] realtime change-record builder registered");
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
import { env } from "~/env.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { logger } from "../logger.server";
|
||||
import { RunChangeNotifier, type ChangeRecordInput } from "./runChangeNotifier.server";
|
||||
|
||||
/**
|
||||
* Process-singleton wiring for the RunChangeNotifier plus the gated convenience functions write sites
|
||||
* delegate to. The notifier is constructed lazily, so `REALTIME_BACKEND_NATIVE_ENABLED=0` (default) opens no Redis connections.
|
||||
*/
|
||||
const nativeBackendEnabled = env.REALTIME_BACKEND_NATIVE_ENABLED === "1";
|
||||
|
||||
function initializeRunChangeNotifier(): RunChangeNotifier {
|
||||
const clusterMode = env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_CLUSTER_MODE_ENABLED === "1";
|
||||
// Sharded pub/sub only works against a cluster; classic pub/sub there would
|
||||
// broadcast every message to every node, so this is what actually shards load.
|
||||
const shardedPubSub =
|
||||
clusterMode && env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_SHARDED_ENABLED === "1";
|
||||
|
||||
const meter = getMeter("realtime-notifier");
|
||||
|
||||
const publishes = meter.createCounter("realtime_notifier.publishes", {
|
||||
description:
|
||||
"Change-record publishes by outcome. Failures are the leading indicator that feeds are degrading to their backstops (pub/sub Redis trouble).",
|
||||
});
|
||||
|
||||
const received = meter.createCounter("realtime_notifier.messages_received", {
|
||||
description: "Raw channel messages received by this instance's subscriber, pre-coalesce.",
|
||||
});
|
||||
|
||||
const delivered = meter.createCounter("realtime_notifier.batches_delivered", {
|
||||
description:
|
||||
"Coalesced batches delivered to listeners. received/batches = the coalesce ratio (how hard a busy env is being collapsed).",
|
||||
});
|
||||
|
||||
const notifier = new RunChangeNotifier({
|
||||
redis: {
|
||||
host: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_HOST,
|
||||
port: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_PORT,
|
||||
username: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_USERNAME,
|
||||
password: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_PASSWORD,
|
||||
tlsDisabled: env.REALTIME_BACKEND_NATIVE_PUBSUB_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode,
|
||||
// One subscriber connection per shard so SSUBSCRIBE routes to the slot owner.
|
||||
...(shardedPubSub ? { clusterOptions: { shardedSubscribers: true } } : {}),
|
||||
},
|
||||
envWakeCoalesceWindowMs: env.REALTIME_BACKEND_NATIVE_ENV_WAKE_COALESCE_WINDOW_MS,
|
||||
shardedPubSub,
|
||||
onPublishResult: (ok) => publishes.add(1, { result: ok ? "ok" : "error" }),
|
||||
onMessageReceived: () => received.add(1),
|
||||
onBatchDelivered: () => delivered.add(1),
|
||||
});
|
||||
|
||||
meter
|
||||
.createObservableGauge("realtime_notifier.active_subscriptions", {
|
||||
description: "Distinct env channels currently subscribed for realtime change notifications.",
|
||||
})
|
||||
.addCallback((result) => result.observe(notifier.activeSubscriptionCount));
|
||||
|
||||
return notifier;
|
||||
}
|
||||
|
||||
/** Lazily construct (and memoize) the notifier singleton. */
|
||||
export function getRunChangeNotifier(): RunChangeNotifier {
|
||||
return singleton("runChangeNotifier", initializeRunChangeNotifier);
|
||||
}
|
||||
|
||||
/** Whether the notifier subsystem is enabled for this process. */
|
||||
export function isRunChangeNotifierEnabled(): boolean {
|
||||
return nativeBackendEnabled;
|
||||
}
|
||||
|
||||
/** Fire-and-forget publish of a run-changed record. No-op (and no notifier construction)
|
||||
* when disabled, so publish sites can call it unconditionally. */
|
||||
export function publishChangeRecord(input: ChangeRecordInput): void {
|
||||
if (!nativeBackendEnabled) {
|
||||
return;
|
||||
}
|
||||
// Publish runs on the run-engine event bus / metadata flush loop; lazy init + encoding happen
|
||||
// before the notifier's own try/catch, so guard the whole call — it must never throw at its caller.
|
||||
try {
|
||||
getRunChangeNotifier().publish(input);
|
||||
} catch (error) {
|
||||
logger.error("[runChangeNotifier] publishChangeRecord threw; dropping notification", { error });
|
||||
}
|
||||
}
|
||||
|
||||
export function publishManyChangeRecords(inputs: ChangeRecordInput[]): void {
|
||||
if (!nativeBackendEnabled) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
getRunChangeNotifier().publishMany(inputs);
|
||||
} catch (error) {
|
||||
logger.error("[runChangeNotifier] publishManyChangeRecords threw; dropping notifications", {
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import {
|
||||
type Prisma,
|
||||
type PrismaClient,
|
||||
type PrismaClientOrTransaction,
|
||||
} from "@trigger.dev/database";
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import { BoundedTtlCache } from "./boundedTtlCache";
|
||||
import { RESERVED_COLUMNS, type RealtimeRunRow } from "./electricStreamProtocol.server";
|
||||
|
||||
/**
|
||||
* RunReader — the pluggable read half of the native-backend realtime feed: ClickHouse is filter-only
|
||||
* (resolves ids), Postgres always hydrates row columns. Owns the `RunHydrator` (by-id) and the
|
||||
* `RunListResolver` interface (the tag/list filter -> id-set seam, implemented over ClickHouse).
|
||||
*/
|
||||
|
||||
/** The TaskRun columns the realtime feed projects (mirrors DEFAULT_ELECTRIC_COLUMNS). */
|
||||
export const RUN_HYDRATOR_SELECT = {
|
||||
id: true,
|
||||
taskIdentifier: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
startedAt: true,
|
||||
delayUntil: true,
|
||||
queuedAt: true,
|
||||
expiredAt: true,
|
||||
completedAt: true,
|
||||
friendlyId: true,
|
||||
number: true,
|
||||
isTest: true,
|
||||
status: true,
|
||||
usageDurationMs: true,
|
||||
costInCents: true,
|
||||
baseCostInCents: true,
|
||||
ttl: true,
|
||||
payload: true,
|
||||
payloadType: true,
|
||||
metadata: true,
|
||||
metadataType: true,
|
||||
output: true,
|
||||
outputType: true,
|
||||
runTags: true,
|
||||
error: true,
|
||||
realtimeStreams: true,
|
||||
} satisfies Prisma.TaskRunSelect;
|
||||
|
||||
/** Columns hydrated regardless of `skipColumns`: `id` keys the row, `updatedAt` drives the offset and working-set diff. */
|
||||
const ALWAYS_HYDRATED_COLUMNS = new Set<string>(["id", "updatedAt", ...RESERVED_COLUMNS]);
|
||||
|
||||
/** Project `RUN_HYDRATOR_SELECT` down to the columns the client didn't skip (plus
|
||||
* the always-needed ones). An empty skip set returns the full select unchanged. */
|
||||
export function buildHydratorSelect(skipColumns: string[] = []): Prisma.TaskRunSelect {
|
||||
if (skipColumns.length === 0) {
|
||||
return RUN_HYDRATOR_SELECT;
|
||||
}
|
||||
const skip = new Set(skipColumns);
|
||||
const select: Record<string, boolean> = {};
|
||||
for (const column of Object.keys(RUN_HYDRATOR_SELECT)) {
|
||||
if (ALWAYS_HYDRATED_COLUMNS.has(column) || !skip.has(column)) {
|
||||
select[column] = true;
|
||||
}
|
||||
}
|
||||
return select as Prisma.TaskRunSelect;
|
||||
}
|
||||
|
||||
export type RunListFilter = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
/** Contains-ANY tag match (OR). Omit/empty for non-tag feeds. */
|
||||
tags?: string[];
|
||||
/** Restrict to a single batch (internal batch id) — the batch feed. */
|
||||
batchId?: string;
|
||||
/** Lower bound on createdAt (the tag-list feed pins this; batch omits it). */
|
||||
createdAtAfter?: Date;
|
||||
/** Hard cap on the result set so a broad filter can't unbound the snapshot. */
|
||||
limit: number;
|
||||
};
|
||||
|
||||
/** Resolves a tag/list filter into the matching run id-set, filter-only (rows hydrated from Postgres by id afterward). ClickHouse impl in `clickHouseRunListResolver.server.ts`. */
|
||||
export interface RunListResolver {
|
||||
resolveMatchingRunIds(filter: RunListFilter): Promise<string[]>;
|
||||
}
|
||||
|
||||
export type RunHydratorOptions = {
|
||||
/** A read-replica Prisma client (`$replica`). Always Postgres. */
|
||||
replica: Pick<PrismaClient, "taskRun">;
|
||||
/** RunStore the reads are routed through; `replica` is passed as the read client. */
|
||||
runStore: RunStore;
|
||||
/** Read-through cache TTL (ms) collapsing duplicate refetches for the same run. Set 0 to disable. Defaults to 250ms. */
|
||||
cacheTtlMs?: number;
|
||||
/** Hard cap on cache entries before expired entries are swept. */
|
||||
maxCacheEntries?: number;
|
||||
};
|
||||
|
||||
const DEFAULT_CACHE_TTL_MS = 250;
|
||||
const DEFAULT_MAX_CACHE_ENTRIES = 5_000;
|
||||
|
||||
/** Hydrates runs by id through the runStore seam (split routing lives in the store, below this file), projected to the realtime columns; concurrent same-run refetches are single-flighted + short-TTL cached. */
|
||||
export class RunHydrator {
|
||||
readonly #inflight = new Map<string, Promise<RealtimeRunRow | null>>();
|
||||
readonly #cache: BoundedTtlCache<RealtimeRunRow | null>;
|
||||
readonly #cacheTtlMs: number;
|
||||
|
||||
constructor(private readonly options: RunHydratorOptions) {
|
||||
this.#cacheTtlMs = options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
|
||||
this.#cache = new BoundedTtlCache(
|
||||
this.#cacheTtlMs,
|
||||
options.maxCacheEntries ?? DEFAULT_MAX_CACHE_ENTRIES
|
||||
);
|
||||
}
|
||||
|
||||
async getRunById(environmentId: string, runId: string): Promise<RealtimeRunRow | null> {
|
||||
const key = `${environmentId}:${runId}`;
|
||||
|
||||
if (this.#cacheTtlMs > 0) {
|
||||
// A cached null is a valid "run not found" hit; only undefined is a miss.
|
||||
const cached = this.#cache.get(key);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
|
||||
const existing = this.#inflight.get(key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const promise = this.#fetch(environmentId, runId).finally(() => this.#inflight.delete(key));
|
||||
this.#inflight.set(key, promise);
|
||||
|
||||
const row = await promise;
|
||||
|
||||
if (this.#cacheTtlMs > 0) {
|
||||
this.#cache.set(key, row);
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Hydrate many runs by id in one query (order not guaranteed); `skipColumns` projects the SELECT so dropped columns aren't shipped. */
|
||||
async hydrateByIds(
|
||||
environmentId: string,
|
||||
ids: string[],
|
||||
skipColumns: string[] = []
|
||||
): Promise<RealtimeRunRow[]> {
|
||||
if (ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const rows = await this.options.runStore.findRuns(
|
||||
{
|
||||
where: {
|
||||
runtimeEnvironmentId: environmentId,
|
||||
id: { in: ids },
|
||||
},
|
||||
select: buildHydratorSelect(skipColumns),
|
||||
},
|
||||
this.options.replica as PrismaClientOrTransaction
|
||||
);
|
||||
return rows as unknown as RealtimeRunRow[];
|
||||
}
|
||||
|
||||
async #fetch(environmentId: string, runId: string): Promise<RealtimeRunRow | null> {
|
||||
const run = await this.options.runStore.findRun(
|
||||
{
|
||||
id: runId,
|
||||
runtimeEnvironmentId: environmentId,
|
||||
},
|
||||
{ select: RUN_HYDRATOR_SELECT },
|
||||
this.options.replica as PrismaClientOrTransaction
|
||||
);
|
||||
|
||||
return (run ?? null) as RealtimeRunRow | null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,742 @@
|
||||
// app/realtime/S2RealtimeStreams.ts
|
||||
import type { UnkeyCache } from "@internal/cache";
|
||||
import type { StreamIngestor, StreamRecord, StreamResponder, StreamResponseOptions } from "./types";
|
||||
import type { LogLevel } from "@trigger.dev/core/logger";
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
import { headerValue } from "@trigger.dev/core/v3";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
// S2's per-record metered-size limit. Verified empirically against
|
||||
// cloud S2: append succeeds at metered=1048576 and 422s at 1048577
|
||||
// with `"record must have metered size less than 1 MiB"` (the "less
|
||||
// than" wording is slightly off — the boundary is inclusive).
|
||||
//
|
||||
// Metered size formula:
|
||||
// metered = 8 (record overhead) + 2*H + Σ(header name + value) + body
|
||||
// where `body` is the unescaped record body length in UTF-8 bytes and
|
||||
// `H` is the number of S2 record headers.
|
||||
//
|
||||
// We attach no record headers (H=0), so the budget reduces to:
|
||||
// 8 + body ≤ 1048576 → body ≤ 1048568
|
||||
export const S2_MAX_METERED_BYTES = 1024 * 1024; // 1 MiB
|
||||
export const S2_RECORD_BASE_OVERHEAD_BYTES = 8;
|
||||
|
||||
/**
|
||||
* Thrown when a record's metered size would exceed S2's hard per-record
|
||||
* limit. Caught by the route handler and surfaced as 413.
|
||||
*/
|
||||
export class S2RecordTooLargeError extends ServiceValidationError {
|
||||
constructor(public readonly meteredBytes: number) {
|
||||
super(
|
||||
`Record metered size ${meteredBytes} bytes exceeds the S2 per-record limit of ${S2_MAX_METERED_BYTES} bytes. Reduce tool-output size or split into smaller parts.`,
|
||||
413
|
||||
);
|
||||
this.name = "S2RecordTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export type S2RealtimeStreamsOptions = {
|
||||
// S2
|
||||
basin: string; // e.g., "my-basin"
|
||||
accessToken: string; // "Bearer" token issued in S2 console
|
||||
streamPrefix?: string; // defaults to ""
|
||||
|
||||
// Custom endpoint for s2-lite (self-hosted)
|
||||
endpoint?: string; // e.g., "http://localhost:4566/v1"
|
||||
|
||||
// Skip access token issuance (s2-lite doesn't support /access-tokens)
|
||||
skipAccessTokens?: boolean;
|
||||
|
||||
// Read behavior
|
||||
s2WaitSeconds?: number;
|
||||
|
||||
flushIntervalMs?: number; // how often to flush buffered chunks (default 200ms)
|
||||
maxRetries?: number; // max number of retries for failed flushes (default 10)
|
||||
|
||||
logger?: Logger;
|
||||
logLevel?: LogLevel;
|
||||
|
||||
accessTokenExpirationInMs?: number;
|
||||
|
||||
cache?: UnkeyCache<{
|
||||
accessToken: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
// Ops the issued S2 access token is scoped to. `trim` is a distinct op
|
||||
// from `append` even though trim records are appended like any other —
|
||||
// without it, `AppendRecord.trim()` 403s with "Operation not permitted".
|
||||
// `chat.agent`'s per-turn trim chain depends on it.
|
||||
//
|
||||
// The fingerprint folds the ops list into the cache key, so any future
|
||||
// scope change auto-invalidates pre-deploy cached tokens.
|
||||
const S2_TOKEN_OPS = ["append", "create-stream", "trim"] as const;
|
||||
const S2_TOKEN_OPS_FINGERPRINT = [...S2_TOKEN_OPS].sort().join(",");
|
||||
|
||||
type S2IssueAccessTokenResponse = { access_token: string };
|
||||
type S2AppendInput = { records: { body: string }[] };
|
||||
type S2AppendAck = {
|
||||
start: { seq_num: number; timestamp: number };
|
||||
end: { seq_num: number; timestamp: number };
|
||||
tail: { seq_num: number; timestamp: number };
|
||||
};
|
||||
|
||||
export class S2RealtimeStreams implements StreamResponder, StreamIngestor {
|
||||
private readonly basin: string;
|
||||
private readonly baseUrl: string;
|
||||
private readonly accountUrl: string;
|
||||
private readonly endpoint?: string;
|
||||
private readonly token: string;
|
||||
private readonly streamPrefix: string;
|
||||
private readonly skipAccessTokens: boolean;
|
||||
|
||||
private readonly s2WaitSeconds: number;
|
||||
|
||||
private readonly flushIntervalMs: number;
|
||||
private readonly maxRetries: number;
|
||||
|
||||
private readonly logger: Logger;
|
||||
private readonly level: LogLevel;
|
||||
|
||||
private readonly accessTokenExpirationInMs: number;
|
||||
|
||||
private readonly cache?: UnkeyCache<{
|
||||
accessToken: string;
|
||||
}>;
|
||||
|
||||
constructor(opts: S2RealtimeStreamsOptions) {
|
||||
this.basin = opts.basin;
|
||||
this.baseUrl = opts.endpoint ?? `https://${this.basin}.b.aws.s2.dev/v1`;
|
||||
this.accountUrl = opts.endpoint ?? `https://aws.s2.dev/v1`;
|
||||
this.endpoint = opts.endpoint;
|
||||
this.token = opts.accessToken;
|
||||
this.streamPrefix = opts.streamPrefix ?? "";
|
||||
this.skipAccessTokens = opts.skipAccessTokens ?? false;
|
||||
|
||||
this.s2WaitSeconds = opts.s2WaitSeconds ?? 60;
|
||||
|
||||
this.flushIntervalMs = opts.flushIntervalMs ?? 200;
|
||||
this.maxRetries = opts.maxRetries ?? 10;
|
||||
|
||||
this.logger = opts.logger ?? new Logger("S2RealtimeStreams", opts.logLevel ?? "info");
|
||||
this.level = opts.logLevel ?? "info";
|
||||
|
||||
this.cache = opts.cache;
|
||||
this.accessTokenExpirationInMs = opts.accessTokenExpirationInMs ?? 60_000 * 60 * 24; // 1 day
|
||||
}
|
||||
|
||||
private toStreamName(runId: string, streamId: string): string {
|
||||
return `${this.streamPrefix}/runs/${runId}/${streamId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an S2 stream name for a `Session`-primitive channel, addressed by
|
||||
* the session's `friendlyId` and the I/O direction. Used by the session
|
||||
* realtime routes to route traffic to `sessions/{friendlyId}/{out|in}`.
|
||||
*/
|
||||
public toSessionStreamName(friendlyId: string, io: "out" | "in"): string {
|
||||
return `${this.streamPrefix}/sessions/${friendlyId}/${io}`;
|
||||
}
|
||||
|
||||
async initializeStream(
|
||||
runId: string,
|
||||
streamId: string
|
||||
): Promise<{ responseHeaders?: Record<string, string> }> {
|
||||
return this.#initializeStreamByName(
|
||||
this.toStreamName(runId, streamId),
|
||||
`/runs/${runId}/${streamId}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize an S2 stream by `(sessionFriendlyId, io)` — mirrors
|
||||
* {@link initializeStream} but addresses the new `sessions/*` key format.
|
||||
*/
|
||||
async initializeSessionStream(
|
||||
friendlyId: string,
|
||||
io: "out" | "in"
|
||||
): Promise<{ responseHeaders?: Record<string, string> }> {
|
||||
return this.#initializeStreamByName(
|
||||
this.toSessionStreamName(friendlyId, io),
|
||||
`/sessions/${friendlyId}/${io}`
|
||||
);
|
||||
}
|
||||
|
||||
async #initializeStreamByName(
|
||||
prefixedName: string,
|
||||
relativeName: string
|
||||
): Promise<{ responseHeaders?: Record<string, string> }> {
|
||||
const accessToken = this.skipAccessTokens
|
||||
? this.token
|
||||
: await this.getS2AccessToken(randomUUID());
|
||||
|
||||
return {
|
||||
responseHeaders: {
|
||||
"X-S2-Access-Token": accessToken,
|
||||
"X-S2-Stream-Name": this.skipAccessTokens ? prefixedName : relativeName,
|
||||
"X-S2-Basin": this.basin,
|
||||
"X-S2-Flush-Interval-Ms": this.flushIntervalMs.toString(),
|
||||
"X-S2-Max-Retries": this.maxRetries.toString(),
|
||||
...(this.endpoint ? { "X-S2-Endpoint": this.endpoint } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
ingestData(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
clientId: string,
|
||||
resumeFromChunk?: number
|
||||
): Promise<Response> {
|
||||
throw new Error("S2 streams are written to S2 via the client, not from the server");
|
||||
}
|
||||
|
||||
async appendPart(part: string, partId: string, runId: string, streamId: string): Promise<void> {
|
||||
await this.#appendPartByName(part, partId, this.toStreamName(runId, streamId));
|
||||
}
|
||||
|
||||
/** Append one record to a `Session` channel; returns its seq (same space as `session-in-event-id`). */
|
||||
async appendPartToSessionStream(
|
||||
part: string,
|
||||
partId: string,
|
||||
friendlyId: string,
|
||||
io: "out" | "in"
|
||||
): Promise<number> {
|
||||
return this.#appendPartByName(part, partId, this.toSessionStreamName(friendlyId, io));
|
||||
}
|
||||
|
||||
async #appendPartByName(part: string, partId: string, s2Stream: string): Promise<number> {
|
||||
this.logger.debug(`S2 appending to stream`, { part, stream: s2Stream });
|
||||
|
||||
const recordBody = JSON.stringify({ data: part, id: partId });
|
||||
const meteredBytes = Buffer.byteLength(recordBody, "utf8") + S2_RECORD_BASE_OVERHEAD_BYTES;
|
||||
if (meteredBytes > S2_MAX_METERED_BYTES) {
|
||||
throw new S2RecordTooLargeError(meteredBytes);
|
||||
}
|
||||
|
||||
const result = await this.s2Append(s2Stream, {
|
||||
records: [{ body: recordBody }],
|
||||
});
|
||||
|
||||
this.logger.debug(`S2 append result`, { result });
|
||||
|
||||
return result.start.seq_num;
|
||||
}
|
||||
|
||||
getLastChunkIndex(runId: string, streamId: string, clientId: string): Promise<number> {
|
||||
throw new Error("S2 streams are written to S2 via the client, not from the server");
|
||||
}
|
||||
|
||||
async readRecords(
|
||||
runId: string,
|
||||
streamId: string,
|
||||
afterSeqNum?: number
|
||||
): Promise<StreamRecord[]> {
|
||||
return this.#readRecordsByName(this.toStreamName(runId, streamId), afterSeqNum);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read records from a `Session`-primitive channel starting after the
|
||||
* given sequence number. Used by the `.wait()` race-check path.
|
||||
*/
|
||||
async readSessionStreamRecords(
|
||||
friendlyId: string,
|
||||
io: "out" | "in",
|
||||
afterSeqNum?: number
|
||||
): Promise<StreamRecord[]> {
|
||||
return this.#readRecordsByName(this.toSessionStreamName(friendlyId, io), afterSeqNum);
|
||||
}
|
||||
|
||||
async #readRecordsByName(s2Stream: string, afterSeqNum?: number): Promise<StreamRecord[]> {
|
||||
const startSeq = afterSeqNum != null ? afterSeqNum + 1 : 0;
|
||||
|
||||
const qs = new URLSearchParams();
|
||||
qs.set("seq_num", String(startSeq));
|
||||
qs.set("clamp", "true");
|
||||
qs.set("wait", "0"); // Non-blocking: return immediately with existing records
|
||||
|
||||
const res = await fetch(
|
||||
`${this.baseUrl}/streams/${encodeURIComponent(s2Stream)}/records?${qs}`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
Accept: "text/event-stream",
|
||||
"S2-Format": "raw",
|
||||
"S2-Basin": this.basin,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (!res.ok) {
|
||||
// Stream may not exist yet (no data sent)
|
||||
if (res.status === 404) {
|
||||
return [];
|
||||
}
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`S2 readRecords failed: ${res.status} ${res.statusText} ${text}`);
|
||||
}
|
||||
|
||||
// Parse the SSE response body to extract records
|
||||
const body = await res.text();
|
||||
return this.parseSSEBatchRecords(body);
|
||||
}
|
||||
|
||||
private parseSSEBatchRecords(sseText: string): StreamRecord[] {
|
||||
const records: StreamRecord[] = [];
|
||||
|
||||
// SSE events are separated by double newlines
|
||||
const events = sseText.split("\n\n").filter((e) => e.trim());
|
||||
|
||||
for (const event of events) {
|
||||
const lines = event.split("\n");
|
||||
let eventType: string | undefined;
|
||||
let data: string | undefined;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("event:")) {
|
||||
eventType = line.slice(6).trim();
|
||||
} else if (line.startsWith("data:")) {
|
||||
data = line.slice(5).trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (eventType === "batch" && data) {
|
||||
try {
|
||||
const parsed = JSON.parse(data) as {
|
||||
records: Array<{
|
||||
body: string;
|
||||
seq_num: number;
|
||||
timestamp: number;
|
||||
headers?: Array<[string, string]>;
|
||||
}>;
|
||||
};
|
||||
|
||||
for (const record of parsed.records) {
|
||||
// S2 command records (trim/fence) have a single header with
|
||||
// empty name. Skip — callers want only data + Trigger control
|
||||
// records.
|
||||
if (record.headers?.[0]?.[0] === "") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Data records carry a JSON envelope; Trigger control records
|
||||
// have an empty body and route via headers. Tolerate non-JSON
|
||||
// bodies so a control record (or a malformed data record)
|
||||
// doesn't take the whole batch down with it.
|
||||
let parsedBody: { data: string; id: string } | undefined;
|
||||
try {
|
||||
parsedBody = JSON.parse(record.body) as { data: string; id: string };
|
||||
} catch {
|
||||
parsedBody = undefined;
|
||||
}
|
||||
records.push({
|
||||
data: parsedBody?.data ?? "",
|
||||
id: parsedBody?.id ?? "",
|
||||
seqNum: record.seq_num,
|
||||
headers: record.headers,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed events
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
// ---------- Serve SSE from S2 ----------
|
||||
|
||||
async streamResponse(
|
||||
request: Request,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
signal: AbortSignal,
|
||||
options?: StreamResponseOptions
|
||||
): Promise<Response> {
|
||||
return this.#streamResponseByName(this.toStreamName(runId, streamId), signal, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve SSE from a `Session`-primitive channel addressed by
|
||||
* `(friendlyId, io)`.
|
||||
*
|
||||
* For `io=out`, peek the tail of the stream. If the most recent
|
||||
* non-command record is a `turn-complete` control record (i.e. the
|
||||
* agent has finished a turn and is either idle-waiting on `.in` or
|
||||
* has exited), no more chunks will arrive without further user
|
||||
* action. We switch the downstream S2 read to `wait=0` (drain
|
||||
* whatever's left, close fast) and set `X-Session-Settled: true` so
|
||||
* the client knows this SSE close is terminal instead of the normal
|
||||
* 60s long-poll cycle.
|
||||
*
|
||||
* The actual tail is now usually an S2 `trim` command record (the
|
||||
* agent appends one after every turn-complete to keep `.out`
|
||||
* bounded). The peek reads two records and walks past the trim to
|
||||
* find the turn-complete underneath.
|
||||
*
|
||||
* Mid-turn tail (streaming UIMessageChunk) falls through to the
|
||||
* long-poll path; a crashed-mid-turn stream is indistinguishable
|
||||
* here and behaves like today (client sees wait=60 close, retries).
|
||||
*/
|
||||
async streamResponseFromSessionStream(
|
||||
request: Request,
|
||||
friendlyId: string,
|
||||
io: "out" | "in",
|
||||
signal: AbortSignal,
|
||||
options?: StreamResponseOptions
|
||||
): Promise<Response> {
|
||||
const s2Stream = this.toSessionStreamName(friendlyId, io);
|
||||
|
||||
let waitSeconds = options?.timeoutInSeconds ?? this.s2WaitSeconds;
|
||||
let settled = false;
|
||||
|
||||
// Only peek + settle when the client opts in via `options.peekSettled`.
|
||||
// Reconnect-on-reload paths (`TriggerChatTransport.reconnectToStream`)
|
||||
// set it; active send-a-message paths don't — otherwise the peek
|
||||
// races the newly-triggered turn's first chunk and the SSE closes
|
||||
// before records land.
|
||||
if (io === "out" && options?.peekSettled) {
|
||||
settled = await this.#peekIsSettled(s2Stream);
|
||||
if (settled) {
|
||||
waitSeconds = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const s2Response = await this.#streamResponseByName(s2Stream, signal, {
|
||||
...options,
|
||||
timeoutInSeconds: waitSeconds,
|
||||
});
|
||||
|
||||
if (!settled) return s2Response;
|
||||
|
||||
const headers = new Headers(s2Response.headers);
|
||||
headers.set("X-Session-Settled", "true");
|
||||
return new Response(s2Response.body, {
|
||||
status: s2Response.status,
|
||||
statusText: s2Response.statusText,
|
||||
headers,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Peek the tail of `.out` and return whether the stream is "settled" —
|
||||
* i.e. the most recent non-command record is a `turn-complete` control
|
||||
* record. The agent appends an S2 `trim` command record immediately
|
||||
* after every turn-complete to keep the stream bounded, so we read two
|
||||
* tail records and walk past any trim command to find the
|
||||
* turn-complete underneath.
|
||||
*/
|
||||
async #peekIsSettled(s2Stream: string): Promise<boolean> {
|
||||
const qs = new URLSearchParams();
|
||||
// `tail_offset=2` rewinds two seq positions; `count=2` caps it to
|
||||
// those two records. At steady state these are `[turn-complete, trim]`.
|
||||
// `wait=0` returns immediately with no long-poll.
|
||||
qs.set("tail_offset", "2");
|
||||
qs.set("count", "2");
|
||||
qs.set("wait", "0");
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${this.baseUrl}/streams/${encodeURIComponent(s2Stream)}/records?${qs}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
Accept: "application/json",
|
||||
"S2-Format": "raw",
|
||||
"S2-Basin": this.basin,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.warn("S2 peek last record: fetch failed", { err, stream: s2Stream });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
// 404: stream has never been written to. 416: range not
|
||||
// satisfiable (empty stream). Both mean "nothing to peek."
|
||||
if (res.status === 404 || res.status === 416) return false;
|
||||
const text = await res.text().catch(() => "");
|
||||
this.logger.warn("S2 peek last record failed", {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
text,
|
||||
stream: s2Stream,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
let records: Array<{
|
||||
body: string;
|
||||
seq_num: number;
|
||||
timestamp: number;
|
||||
headers?: Array<[string, string]>;
|
||||
}>;
|
||||
try {
|
||||
const json = (await res.json()) as {
|
||||
records?: Array<{
|
||||
body: string;
|
||||
seq_num: number;
|
||||
timestamp: number;
|
||||
headers?: Array<[string, string]>;
|
||||
}>;
|
||||
};
|
||||
records = json.records ?? [];
|
||||
} catch (err) {
|
||||
this.logger.warn("S2 peek last record: parse failed", { err, stream: s2Stream });
|
||||
return false;
|
||||
}
|
||||
|
||||
// Walk from most-recent backward, skipping S2 command records
|
||||
// (`headers[0][0] === ""`). The first non-command record is the
|
||||
// real tail — settled iff its `trigger-control` header is
|
||||
// `turn-complete`.
|
||||
for (let i = records.length - 1; i >= 0; i--) {
|
||||
const record = records[i]!;
|
||||
if (record.headers?.[0]?.[0] === "") {
|
||||
continue;
|
||||
}
|
||||
const controlValue = headerValue(record.headers, "trigger-control");
|
||||
return controlValue === "turn-complete";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async #streamResponseByName(
|
||||
s2Stream: string,
|
||||
signal: AbortSignal,
|
||||
options?: StreamResponseOptions
|
||||
): Promise<Response> {
|
||||
const startSeq = this.parseLastEventId(options?.lastEventId);
|
||||
|
||||
this.logger.info(`S2 streaming records from stream`, { stream: s2Stream, startSeq });
|
||||
|
||||
// Request SSE stream from S2 and return it directly
|
||||
const s2Response = await this.s2StreamRecords(s2Stream, {
|
||||
seq_num: startSeq ?? 0,
|
||||
clamp: true,
|
||||
wait: options?.timeoutInSeconds ?? this.s2WaitSeconds, // S2 will keep the connection open and stream new records
|
||||
signal, // Pass abort signal so S2 connection is cleaned up when client disconnects
|
||||
});
|
||||
|
||||
// Return S2's SSE response directly to the client
|
||||
return s2Response;
|
||||
}
|
||||
|
||||
// ---------- Internals: S2 REST ----------
|
||||
private async s2Append(stream: string, body: S2AppendInput): Promise<S2AppendAck> {
|
||||
// POST /v1/streams/{stream}/records (JSON).
|
||||
//
|
||||
// Retries transient failures (network errors and 5xx) up to 3 times with
|
||||
// exponential backoff. Undici's "fetch failed" errors observed locally
|
||||
// are pre-connection (DNS/TCP) so the request never reaches S2, making
|
||||
// retry safe — the alternative is a 500 surfacing to the SDK transport,
|
||||
// which then retries the whole `/in/append` round-trip and pollutes
|
||||
// logs. 4xx are not retried (genuine client errors).
|
||||
const url = `${this.baseUrl}/streams/${encodeURIComponent(stream)}/records`;
|
||||
const init: RequestInit = {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
"Content-Type": "application/json",
|
||||
"S2-Format": "raw",
|
||||
"S2-Basin": this.basin,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
};
|
||||
|
||||
const maxAttempts = 3;
|
||||
const backoffsMs = [100, 250, 600];
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
// The `try` only wraps `fetch` — once we have a Response we handle status
|
||||
// outside the catch, so a 4xx throw can't be swallowed and retried.
|
||||
let res: Response | undefined;
|
||||
try {
|
||||
res = await fetch(url, init);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
|
||||
if (res) {
|
||||
if (res.ok) {
|
||||
return (await res.json()) as S2AppendAck;
|
||||
}
|
||||
const text = await res.text().catch(() => "");
|
||||
const httpError = new Error(`S2 append failed: ${res.status} ${res.statusText} ${text}`);
|
||||
if (res.status >= 400 && res.status < 500) {
|
||||
// 4xx — caller-side problem (auth, malformed body, closed stream).
|
||||
// Retrying won't help.
|
||||
throw httpError;
|
||||
}
|
||||
// 5xx — retryable.
|
||||
lastError = httpError;
|
||||
}
|
||||
|
||||
const isLastAttempt = attempt === maxAttempts - 1;
|
||||
const diagnostics = describeFetchError(lastError);
|
||||
if (isLastAttempt) {
|
||||
this.logger.error("S2 append failed after retries", {
|
||||
stream,
|
||||
attempts: maxAttempts,
|
||||
...diagnostics,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
this.logger.warn("S2 append transient failure, retrying", {
|
||||
stream,
|
||||
attempt: attempt + 1,
|
||||
nextDelayMs: backoffsMs[attempt],
|
||||
...diagnostics,
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, backoffsMs[attempt]));
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
||||
}
|
||||
|
||||
private async getS2AccessToken(id: string): Promise<string> {
|
||||
if (!this.cache) {
|
||||
return this.s2IssueAccessToken(id);
|
||||
}
|
||||
|
||||
// Cache key includes basin so per-org basins never collide on
|
||||
// cached tokens, and the ops fingerprint so a scope change in code
|
||||
// (e.g. adding `trim` in #3644) auto-invalidates pre-deploy entries
|
||||
// instead of returning stale tokens for up to 24h.
|
||||
const cacheKey = `${this.basin}:${this.streamPrefix}:${S2_TOKEN_OPS_FINGERPRINT}`;
|
||||
const result = await this.cache.accessToken.swr(cacheKey, async () => {
|
||||
return this.s2IssueAccessToken(id);
|
||||
});
|
||||
|
||||
if (!result.val) {
|
||||
throw new Error("Failed to get S2 access token");
|
||||
}
|
||||
|
||||
return result.val;
|
||||
}
|
||||
|
||||
private async s2IssueAccessToken(id: string): Promise<string> {
|
||||
// POST /v1/access-tokens
|
||||
const res = await fetch(`${this.accountUrl}/access-tokens`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
id,
|
||||
scope: {
|
||||
basins: {
|
||||
exact: this.basin,
|
||||
},
|
||||
ops: [...S2_TOKEN_OPS],
|
||||
streams: {
|
||||
prefix: this.streamPrefix,
|
||||
},
|
||||
},
|
||||
expires_at: new Date(Date.now() + this.accessTokenExpirationInMs).toISOString(),
|
||||
auto_prefix_streams: true,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`S2 issue access token failed: ${res.status} ${res.statusText} ${text}`);
|
||||
}
|
||||
const data = (await res.json()) as S2IssueAccessTokenResponse;
|
||||
return data.access_token;
|
||||
}
|
||||
|
||||
private async s2StreamRecords(
|
||||
stream: string,
|
||||
opts: {
|
||||
seq_num?: number;
|
||||
clamp?: boolean;
|
||||
wait?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
): Promise<Response> {
|
||||
// GET /v1/streams/{stream}/records with Accept: text/event-stream for SSE streaming
|
||||
const qs = new URLSearchParams();
|
||||
if (opts.seq_num != null) qs.set("seq_num", String(opts.seq_num));
|
||||
if (opts.clamp != null) qs.set("clamp", String(opts.clamp));
|
||||
if (opts.wait != null) qs.set("wait", String(opts.wait));
|
||||
|
||||
const res = await fetch(`${this.baseUrl}/streams/${encodeURIComponent(stream)}/records?${qs}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.token}`,
|
||||
Accept: "text/event-stream",
|
||||
"S2-Format": "raw",
|
||||
"S2-Basin": this.basin,
|
||||
},
|
||||
signal: opts.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`S2 stream failed: ${res.status} ${res.statusText} ${text}`);
|
||||
}
|
||||
|
||||
const headers = new Headers(res.headers);
|
||||
headers.set("X-Stream-Version", "v2");
|
||||
headers.set("Access-Control-Expose-Headers", "*");
|
||||
|
||||
return new Response(res.body, {
|
||||
headers,
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
});
|
||||
}
|
||||
|
||||
private parseLastEventId(lastEventId?: string): number | undefined {
|
||||
if (!lastEventId) return undefined;
|
||||
// tolerate formats like "1699999999999-5" (take leading digits)
|
||||
const digits = lastEventId.split("-")[0];
|
||||
const n = Number(digits);
|
||||
return Number.isFinite(n) && n >= 0 ? n + 1 : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Pulls the underlying network error out of undici's generic "fetch failed".
|
||||
// undici sets `error.cause` to either a SystemError-shaped object with `code`
|
||||
// (e.g. `ECONNRESET`, `UND_ERR_SOCKET`, `ETIMEDOUT`), `errno`, and `syscall`,
|
||||
// or — for happy-eyeballs / multi-address connect attempts — an
|
||||
// `AggregateError` whose `errors[]` each carry their own code. Surfacing
|
||||
// those tells us whether failures are pre-connection (DNS / TCP), mid-stream
|
||||
// socket resets, or genuine S2 server errors.
|
||||
function describeFetchError(err: unknown): Record<string, unknown> {
|
||||
if (!(err instanceof Error)) {
|
||||
return { error: String(err) };
|
||||
}
|
||||
const out: Record<string, unknown> = {
|
||||
error: err.message,
|
||||
name: err.name,
|
||||
};
|
||||
const cause = (err as { cause?: unknown }).cause;
|
||||
if (cause && typeof cause === "object") {
|
||||
const c = cause as Record<string, unknown>;
|
||||
if (typeof c.code === "string") out.causeCode = c.code;
|
||||
if (typeof c.errno === "number" || typeof c.errno === "string") out.causeErrno = c.errno;
|
||||
if (typeof c.syscall === "string") out.causeSyscall = c.syscall;
|
||||
if (typeof c.message === "string") out.causeMessage = c.message;
|
||||
if (Array.isArray(c.errors)) {
|
||||
out.causeErrors = c.errors
|
||||
.filter((e: unknown): e is Error => e instanceof Error)
|
||||
.map((e) => ({
|
||||
message: e.message,
|
||||
code: (e as { code?: unknown }).code,
|
||||
syscall: (e as { syscall?: unknown }).syscall,
|
||||
address: (e as { address?: unknown }).address,
|
||||
port: (e as { port?: unknown }).port,
|
||||
}));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
import type { Session, TaskRunStatus } from "@trigger.dev/database";
|
||||
import { SessionTriggerConfig as SessionTriggerConfigZod } from "@trigger.dev/core/v3";
|
||||
import type { z } from "zod";
|
||||
import { prisma, $replica } from "~/db.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server";
|
||||
import { TriggerTaskService } from "~/v3/services/triggerTask.server";
|
||||
import { isFinalRunStatus } from "~/v3/taskStatus";
|
||||
|
||||
/**
|
||||
* Schema for `Session.triggerConfig` (stored as JSONB). The wire-format
|
||||
* source of truth lives in `@trigger.dev/core/v3` as `SessionTriggerConfig`;
|
||||
* we re-export it here for the trigger machinery to validate on read.
|
||||
*
|
||||
* `basePayload` carries the customer's wire payload (for chat.agent:
|
||||
* `{ chatId, ...clientData, idleTimeoutInSeconds? }`). Runtime fields
|
||||
* specific to a particular trigger (e.g. `trigger: "trigger" | "preload"`,
|
||||
* an `isContinuation` flag) come in via the `payloadOverrides` argument
|
||||
* to `ensureRunForSession` and shallow-merge on top of `basePayload`.
|
||||
*/
|
||||
export const SessionTriggerConfigSchema = SessionTriggerConfigZod;
|
||||
|
||||
export type SessionTriggerConfig = z.infer<typeof SessionTriggerConfigSchema>;
|
||||
|
||||
export type EnsureRunReason = "initial" | "continuation" | "upgrade" | "manual";
|
||||
|
||||
/**
|
||||
* Hard cap on how many times `ensureRunForSession` will recurse on the
|
||||
* pathological "we lost the claim race AND the winner's run was already
|
||||
* terminal" path. In practice progress through the run engine bounds
|
||||
* this, but a misconfigured task that crashes before it can be dequeued
|
||||
* could otherwise loop without limit. After this many attempts we
|
||||
* surface `SessionRunManagerError` so the caller can 5xx instead of
|
||||
* blowing the stack.
|
||||
*/
|
||||
const ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS = 3;
|
||||
|
||||
type EnsureRunForSessionParams = {
|
||||
/**
|
||||
* Session row to operate on. Caller is responsible for the env match —
|
||||
* we don't re-check `runtimeEnvironmentId` against `environment.id`.
|
||||
*
|
||||
* `friendlyId` is used to pre-populate `payload.sessionId` on the new
|
||||
* run so the agent's `chat.agent` boot path can attach to `session.in/.out`
|
||||
* without a control-plane round-trip. `currentRunId` is also forwarded
|
||||
* as `payload.previousRunId` (with `continuation: true`) when the prior
|
||||
* run is dead, so the agent's boot gate triggers snapshot.read + replay
|
||||
* instead of treating the run as a fresh chat.
|
||||
*/
|
||||
session: Pick<
|
||||
Session,
|
||||
"id" | "friendlyId" | "taskIdentifier" | "triggerConfig" | "currentRunId" | "currentRunVersion"
|
||||
>;
|
||||
environment: AuthenticatedEnvironment;
|
||||
reason: EnsureRunReason;
|
||||
/**
|
||||
* Shallow-merged on top of `triggerConfig.basePayload`. Runtime fields
|
||||
* only — caller-controlled data that varies per trigger (`trigger:
|
||||
* "preload"` vs `"trigger"`, etc).
|
||||
*/
|
||||
payloadOverrides?: Record<string, unknown>;
|
||||
/**
|
||||
* @internal Recursion-guard counter for the lost-claim-race retry path.
|
||||
* Public callers should leave this unset; the function recurses with
|
||||
* an incremented value on the pathological "winner's run was already
|
||||
* terminal" branch and throws once it exceeds
|
||||
* {@link ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS}.
|
||||
*/
|
||||
_attempt?: number;
|
||||
};
|
||||
|
||||
export type EnsureRunResult = {
|
||||
runId: string;
|
||||
/** True if this call triggered a fresh run; false if it reused an alive existing one. */
|
||||
triggered: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Idempotently make sure the session has a live run.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. If `currentRunId` is set, probe its status. Alive → return as-is.
|
||||
* 2. Trigger a new run upfront (cheap to cancel if we lose the race).
|
||||
* 3. Atomic claim via `updateMany` keyed on `currentRunVersion`.
|
||||
* - Won: return new runId, record SessionRun audit row.
|
||||
* - Lost: cancel our triggered run, re-read session, reuse winner's
|
||||
* run if alive. If pathological (winner's run already terminal),
|
||||
* recurse.
|
||||
*
|
||||
* No DB lock is held across the trigger call. Wasted-trigger window is
|
||||
* the rare multi-tab race on a dead run; cancel cost is negligible and
|
||||
* the run-engine handles it gracefully.
|
||||
*/
|
||||
export async function ensureRunForSession(
|
||||
params: EnsureRunForSessionParams
|
||||
): Promise<EnsureRunResult> {
|
||||
const { session, environment, reason, payloadOverrides, _attempt = 1 } = params;
|
||||
|
||||
if (_attempt > ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS) {
|
||||
throw new SessionRunManagerError(
|
||||
`ensureRunForSession exceeded ${ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS} attempts for session ${session.id} — every triggered run reached a terminal state before claim could resolve`
|
||||
);
|
||||
}
|
||||
|
||||
// 1. Probe currentRunId.
|
||||
let priorDeadRunFriendlyId: string | undefined;
|
||||
if (session.currentRunId) {
|
||||
let probe = await getRunStatusAndFriendlyId(session.currentRunId);
|
||||
if (!probe) {
|
||||
// Replica miss on a row we just observed via `currentRunId` — the
|
||||
// run was likely triggered moments ago and hasn't replicated yet.
|
||||
// Re-probe the writer BEFORE deciding liveness: treating a lagging
|
||||
// replica as "row vanished" double-triggers the session (a fast
|
||||
// first append after session create races the replica apply delay
|
||||
// and spawns a second live run consuming the same `.in`).
|
||||
probe = await runStore.findRun(
|
||||
{ id: session.currentRunId },
|
||||
{ select: { status: true, friendlyId: true } },
|
||||
prisma
|
||||
);
|
||||
}
|
||||
if (probe && !isFinalRunStatus(probe.status)) {
|
||||
return { runId: session.currentRunId, triggered: false };
|
||||
}
|
||||
// Either the row vanished on the writer too (probe null) or its status
|
||||
// is final. Either way the prior run isn't going to consume new
|
||||
// appends — but the session may still hold conversation state on
|
||||
// `session.out` and an S3 snapshot keyed on `session.friendlyId`.
|
||||
// Forward the prior run's public-form id (friendlyId — same shape as
|
||||
// `ctx.run.id`) to the agent as `previousRunId` so its boot gate flips
|
||||
// `couldHavePriorState` and replays the persisted state instead of
|
||||
// treating this as a fresh chat. See `chat.agent`'s boot orchestration
|
||||
// in `packages/trigger-sdk/src/v3/ai.ts`.
|
||||
priorDeadRunFriendlyId = probe?.friendlyId ?? session.currentRunId;
|
||||
}
|
||||
|
||||
// 2. Validate config + trigger upfront. Continuation overrides
|
||||
// (`continuation`, `previousRunId`) are derived from session state above
|
||||
// and merged AFTER caller-supplied overrides — caller can't accidentally
|
||||
// unset them on a session that has had a prior run, but can still
|
||||
// override `trigger`/`metadata` etc. `sessionId` is always set so the
|
||||
// agent doesn't need a control-plane round-trip to look up the session
|
||||
// friendlyId from `payload.chatId`.
|
||||
// Continuation overrides strip the basePayload's first-run-only fields
|
||||
// so a continuation run doesn't inherit a stale boot payload. The Session
|
||||
// row's `triggerConfig.basePayload` is captured at create-time and used
|
||||
// verbatim for every Run we trigger; if the customer included `message`
|
||||
// / `messages` / `trigger: "submit-message"` to make the FIRST run boot
|
||||
// straight into a first turn (via `chat.createStartSessionAction`), those
|
||||
// values stick around and get replayed on every continuation. With
|
||||
// `continuation: true` and `message`/`messages` cleared, the SDK boot
|
||||
// path enters its continuation-wait branch and waits for the next
|
||||
// session.in record before running a turn.
|
||||
const continuationOverrides: Record<string, unknown> = {
|
||||
sessionId: session.friendlyId,
|
||||
...(priorDeadRunFriendlyId !== undefined
|
||||
? {
|
||||
continuation: true,
|
||||
previousRunId: priorDeadRunFriendlyId,
|
||||
// Clear sticky boot-payload fields so the new run waits for the
|
||||
// next session.in record instead of re-processing whatever was
|
||||
// in the original `createStartSessionAction({ basePayload })`.
|
||||
message: undefined,
|
||||
messages: undefined,
|
||||
trigger: undefined,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const mergedPayloadOverrides: Record<string, unknown> = {
|
||||
...(payloadOverrides ?? {}),
|
||||
...continuationOverrides,
|
||||
};
|
||||
|
||||
const config = SessionTriggerConfigSchema.parse(session.triggerConfig);
|
||||
const triggered = await triggerSessionRun({
|
||||
session,
|
||||
config,
|
||||
environment,
|
||||
payloadOverrides: mergedPayloadOverrides,
|
||||
});
|
||||
|
||||
// 3. Try to claim the slot atomically.
|
||||
const claim = await prisma.session.updateMany({
|
||||
where: {
|
||||
id: session.id,
|
||||
currentRunVersion: session.currentRunVersion,
|
||||
},
|
||||
data: {
|
||||
currentRunId: triggered.id,
|
||||
currentRunVersion: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
if (claim.count === 1) {
|
||||
// Won. Audit the SessionRun. Best-effort — failure here doesn't
|
||||
// invalidate the live run, just leaves a missing audit row.
|
||||
prisma.sessionRun
|
||||
.create({
|
||||
data: { sessionId: session.id, runId: triggered.id, reason },
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.warn("Failed to record SessionRun audit row", {
|
||||
sessionId: session.id,
|
||||
runId: triggered.id,
|
||||
reason,
|
||||
error,
|
||||
});
|
||||
});
|
||||
|
||||
return { runId: triggered.id, triggered: true };
|
||||
}
|
||||
|
||||
// 4. Lost the race. Cancel our triggered run; reuse the winner's.
|
||||
cancelLostRaceRun(triggered.id, environment).catch((error) => {
|
||||
logger.warn("Failed to cancel lost-race session run", {
|
||||
sessionId: session.id,
|
||||
runId: triggered.id,
|
||||
error,
|
||||
});
|
||||
});
|
||||
|
||||
// Read-after-write: the winner just wrote `currentRunId` /
|
||||
// `currentRunVersion` on the writer. Reading from `$replica` could
|
||||
// return pre-race state and cause us to recurse with the same stale
|
||||
// version, losing the next claim, until we exhaust max attempts.
|
||||
const fresh = await prisma.session.findFirst({
|
||||
where: { id: session.id },
|
||||
select: {
|
||||
id: true,
|
||||
friendlyId: true,
|
||||
taskIdentifier: true,
|
||||
triggerConfig: true,
|
||||
currentRunId: true,
|
||||
currentRunVersion: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!fresh) {
|
||||
// Session vanished mid-flight. Surface as an error — caller decides
|
||||
// whether to 404 or retry.
|
||||
throw new SessionRunManagerError(`Session ${session.id} not found after lost claim race`);
|
||||
}
|
||||
|
||||
if (fresh.currentRunId) {
|
||||
// Same read-after-write reason as the `fresh` reload above: the winner
|
||||
// just wrote `currentRunId` on the writer, so probe the writer too —
|
||||
// the replica may not have the run row yet, and a missed probe forces
|
||||
// another trigger+recurse until `ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS`.
|
||||
const probe = await runStore.findRun(
|
||||
{ id: fresh.currentRunId },
|
||||
{ select: { status: true, friendlyId: true } },
|
||||
prisma
|
||||
);
|
||||
if (probe && !isFinalRunStatus(probe.status)) {
|
||||
return { runId: fresh.currentRunId, triggered: false };
|
||||
}
|
||||
}
|
||||
|
||||
// Pathological: winner's run already terminal. Recurse with the fresh
|
||||
// version. Bounded by `ENSURE_RUN_FOR_SESSION_MAX_ATTEMPTS` so a task
|
||||
// that always crashes before being dequeued surfaces as an error
|
||||
// instead of a stack overflow.
|
||||
return ensureRunForSession({
|
||||
session: fresh,
|
||||
environment,
|
||||
reason,
|
||||
payloadOverrides,
|
||||
_attempt: _attempt + 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a single run for a session. Builds `TriggerTaskRequestBody`
|
||||
* by shallow-merging `payloadOverrides` over `config.basePayload` and
|
||||
* threading `config`'s machine/queue/tags through the trigger options.
|
||||
*/
|
||||
async function triggerSessionRun(params: {
|
||||
session: Pick<Session, "id" | "taskIdentifier">;
|
||||
config: SessionTriggerConfig;
|
||||
environment: AuthenticatedEnvironment;
|
||||
payloadOverrides?: Record<string, unknown>;
|
||||
}): Promise<{ id: string; friendlyId: string }> {
|
||||
const { session, config, environment, payloadOverrides } = params;
|
||||
|
||||
const payload = {
|
||||
...config.basePayload,
|
||||
...(config.idleTimeoutInSeconds !== undefined
|
||||
? { idleTimeoutInSeconds: config.idleTimeoutInSeconds }
|
||||
: {}),
|
||||
...(payloadOverrides ?? {}),
|
||||
};
|
||||
|
||||
const body = {
|
||||
payload,
|
||||
context: {},
|
||||
options: {
|
||||
...(config.machine ? { machine: config.machine as never } : {}),
|
||||
...(config.queue ? { queue: { name: config.queue } } : {}),
|
||||
...(config.tags ? { tags: config.tags } : {}),
|
||||
...(config.maxAttempts !== undefined ? { maxAttempts: config.maxAttempts } : {}),
|
||||
...(config.maxDuration !== undefined ? { maxDuration: config.maxDuration } : {}),
|
||||
...(config.lockToVersion ? { lockToVersion: config.lockToVersion } : {}),
|
||||
...(config.region ? { region: config.region } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
const service = new TriggerTaskService();
|
||||
const result = await service.call(session.taskIdentifier, environment, body, {
|
||||
triggerSource: "session",
|
||||
triggerAction: "trigger",
|
||||
});
|
||||
|
||||
if (!result) {
|
||||
throw new SessionRunManagerError(
|
||||
`TriggerTaskService returned no result for taskIdentifier=${session.taskIdentifier}`
|
||||
);
|
||||
}
|
||||
|
||||
return { id: result.run.id, friendlyId: result.run.friendlyId };
|
||||
}
|
||||
|
||||
type SwapSessionRunParams = {
|
||||
/**
|
||||
* Session row to swap. `friendlyId` is forwarded as `payload.sessionId`
|
||||
* on the new run so the agent attaches to `session.in/.out` without a
|
||||
* control-plane round-trip (same convention as
|
||||
* {@link EnsureRunForSessionParams}).
|
||||
*/
|
||||
session: Pick<
|
||||
Session,
|
||||
"id" | "friendlyId" | "taskIdentifier" | "triggerConfig" | "currentRunId" | "currentRunVersion"
|
||||
>;
|
||||
/**
|
||||
* The run requesting the swap. Optimistic claim requires
|
||||
* `Session.currentRunId === callingRunId` so the swap can't clobber
|
||||
* a run triggered out-of-band (e.g. a parallel `.in/append` probe
|
||||
* that already replaced the dead run).
|
||||
*
|
||||
* Also forwarded as `payload.previousRunId` on the new run alongside
|
||||
* `continuation: true` — every swap is a continuation by construction
|
||||
* (`chat.requestUpgrade` / `chat.endRun` deliberately hand off prior
|
||||
* conversation state to a new run), so the agent's boot gate flips
|
||||
* `couldHavePriorState` and replays the snapshot + session.out tail.
|
||||
*/
|
||||
callingRunId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
reason: EnsureRunReason;
|
||||
payloadOverrides?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type SwapSessionRunResult = {
|
||||
/** runId of the newly-triggered run that has taken over the session. */
|
||||
runId: string;
|
||||
/**
|
||||
* False when the swap was preempted (currentRunId is no longer the
|
||||
* calling run). The caller should treat this as "someone else
|
||||
* already moved on" — exit cleanly without expecting to drive the
|
||||
* next run.
|
||||
*/
|
||||
swapped: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Force-swap the session to a freshly-triggered run, regardless of
|
||||
* whether the current run is alive. Called by `end-and-continue` when
|
||||
* the running agent wants a clean handoff (typically version upgrade).
|
||||
*
|
||||
* Differs from `ensureRunForSession`: never reuses the current run.
|
||||
* The optimistic claim is keyed on `currentRunId === callingRunId`, so
|
||||
* a parallel append-time probe that already swapped to a different
|
||||
* run wins the race and `swapped: false` is surfaced.
|
||||
*/
|
||||
export async function swapSessionRun(params: SwapSessionRunParams): Promise<SwapSessionRunResult> {
|
||||
const { session, callingRunId, environment, reason, payloadOverrides } = params;
|
||||
|
||||
// `callingRunId` is the internal cuid (`Session.currentRunId` stores
|
||||
// cuid; the route handler resolves the wire's friendlyId before passing
|
||||
// it here). The agent's `previousRunId` is customer-visible and must
|
||||
// match the public `run_*` form exposed via `ctx.run.id` — resolve
|
||||
// before forwarding.
|
||||
const callingRunFriendlyId = await resolveRunFriendlyId(callingRunId);
|
||||
|
||||
// Continuation overrides — unconditionally set on swap. Unlike
|
||||
// `ensureRunForSession`, there's no dead-run-detection branch here:
|
||||
// every swap is a deliberate handoff from `callingRunId` (which owned
|
||||
// prior conversation state) to a fresh run. Merged AFTER caller-supplied
|
||||
// overrides so a caller can't accidentally unset them.
|
||||
//
|
||||
// Sticky boot-payload fields (`message` / `messages` / `trigger`) are
|
||||
// cleared here for the same reason as in `ensureRunForSession`: the
|
||||
// Session's basePayload is captured at create-time and replays on every
|
||||
// continuation if not stripped. See the comment in `ensureRunForSession`.
|
||||
const mergedPayloadOverrides: Record<string, unknown> = {
|
||||
...(payloadOverrides ?? {}),
|
||||
sessionId: session.friendlyId,
|
||||
continuation: true,
|
||||
previousRunId: callingRunFriendlyId,
|
||||
message: undefined,
|
||||
messages: undefined,
|
||||
trigger: undefined,
|
||||
};
|
||||
|
||||
const config = SessionTriggerConfigSchema.parse(session.triggerConfig);
|
||||
const triggered = await triggerSessionRun({
|
||||
session,
|
||||
config,
|
||||
environment,
|
||||
payloadOverrides: mergedPayloadOverrides,
|
||||
});
|
||||
|
||||
const claim = await prisma.session.updateMany({
|
||||
where: {
|
||||
id: session.id,
|
||||
currentRunId: callingRunId,
|
||||
currentRunVersion: session.currentRunVersion,
|
||||
},
|
||||
data: {
|
||||
currentRunId: triggered.id,
|
||||
currentRunVersion: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
if (claim.count === 1) {
|
||||
prisma.sessionRun
|
||||
.create({
|
||||
data: { sessionId: session.id, runId: triggered.id, reason },
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.warn("Failed to record SessionRun audit row", {
|
||||
sessionId: session.id,
|
||||
runId: triggered.id,
|
||||
reason,
|
||||
error,
|
||||
});
|
||||
});
|
||||
return { runId: triggered.id, swapped: true };
|
||||
}
|
||||
|
||||
// Lost the race — someone else already swapped to a new run. Cancel
|
||||
// ours, surface the existing winner.
|
||||
cancelLostRaceRun(triggered.id, environment).catch((error) => {
|
||||
logger.warn("Failed to cancel preempted swap run", {
|
||||
sessionId: session.id,
|
||||
runId: triggered.id,
|
||||
error,
|
||||
});
|
||||
});
|
||||
|
||||
// Read-after-write: the winner's swap was just committed on the
|
||||
// writer. A replica read could return the pre-swap `currentRunId`
|
||||
// (often `callingRunId` itself), which would tell the caller it is
|
||||
// still the canonical run when in fact a different run has taken
|
||||
// over.
|
||||
const fresh = await prisma.session.findFirst({
|
||||
where: { id: session.id },
|
||||
select: { currentRunId: true },
|
||||
});
|
||||
|
||||
// Mirror `ensureRunForSession`'s "session vanished" branch: if we
|
||||
// can't find the row (or it has no current run) on the writer right
|
||||
// after losing the race, surface as an error rather than handing back
|
||||
// `callingRunId` with `swapped: false` — that would tell the caller
|
||||
// it's still the canonical run when in fact we don't know who is.
|
||||
if (!fresh?.currentRunId) {
|
||||
throw new SessionRunManagerError(
|
||||
`Session ${session.id} has no currentRunId after preempted swap`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
runId: fresh.currentRunId,
|
||||
swapped: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function getRunStatusAndFriendlyId(
|
||||
runId: string
|
||||
): Promise<{ status: TaskRunStatus; friendlyId: string } | null> {
|
||||
// Use the read replica — this is a hot-path probe and stale-by-ms is
|
||||
// fine. The append handler re-checks if it ends up reusing the runId.
|
||||
// `friendlyId` is fetched alongside `status` so the dead-run-detection
|
||||
// branch in `ensureRunForSession` can forward the public-form id as
|
||||
// `payload.previousRunId` without a second read. `Session.currentRunId`
|
||||
// stores the internal cuid; the agent's wire / customer hooks expose
|
||||
// the friendlyId via `ctx.run.id`, so consistency matters.
|
||||
const row = await runStore.findRun(
|
||||
{ id: runId },
|
||||
{ select: { status: true, friendlyId: true } },
|
||||
$replica
|
||||
);
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a TaskRun cuid to its friendlyId. Used by `swapSessionRun` to
|
||||
* forward the calling run's public-form id as `payload.previousRunId` on
|
||||
* the new run. Falls back to the cuid on lookup miss so the swap doesn't
|
||||
* fail just because the read replica hasn't caught up — the agent only
|
||||
* uses `previousRunId` for customer-visible bookkeeping (e.g.
|
||||
* `runs.retrieve(previousRunId)`), so a stale-but-non-null value is
|
||||
* acceptable degraded behavior.
|
||||
*/
|
||||
async function resolveRunFriendlyId(runId: string): Promise<string> {
|
||||
const row = await runStore.findRun({ id: runId }, { select: { friendlyId: true } }, $replica);
|
||||
return row?.friendlyId ?? runId;
|
||||
}
|
||||
|
||||
async function cancelLostRaceRun(
|
||||
runId: string,
|
||||
environment: AuthenticatedEnvironment
|
||||
): Promise<void> {
|
||||
const service = new CancelTaskRunService();
|
||||
// Read-after-write: the run was just triggered on the writer, so go
|
||||
// through `prisma`. A `$replica` miss here would silently no-op the
|
||||
// cancel and leak an orphan run that no session is going to claim.
|
||||
const run = await runStore.findRun({ id: runId }, prisma);
|
||||
if (!run) return;
|
||||
await service.call(run, { reason: "Lost session-run claim race" });
|
||||
}
|
||||
|
||||
export class SessionRunManagerError extends Error {
|
||||
readonly name = "SessionRunManagerError";
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { PrismaClient, Session } from "@trigger.dev/database";
|
||||
import type { SessionItem } from "@trigger.dev/core/v3";
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import { $replica, prisma } from "~/db.server";
|
||||
import { runStore as defaultRunStore } from "~/v3/runStore.server";
|
||||
|
||||
/**
|
||||
* Prefix that {@link SessionId.generate} attaches to every Session friendlyId.
|
||||
* Used to distinguish friendlyId lookups (`session_abc...`) from externalId
|
||||
* lookups on the public `GET /api/v1/sessions/:session` route.
|
||||
*/
|
||||
const SESSION_FRIENDLY_ID_PREFIX = "session_";
|
||||
|
||||
/**
|
||||
* Resolve a session from a URL path parameter that may contain either a
|
||||
* friendlyId (`session_abc...`) or a user-supplied externalId.
|
||||
*
|
||||
* Disambiguated by prefix: values starting with `session_` are treated as
|
||||
* friendlyIds, anything else is looked up against `externalId` scoped to
|
||||
* the caller's environment.
|
||||
*/
|
||||
// CONTROL-PLANE: `Session` lives on the control-plane DB; these reads are NOT
|
||||
// routed to run-ops read-through — only the `TaskRun` currentRunId resolves in
|
||||
// this file are run-ops read-through routed.
|
||||
export async function resolveSessionByIdOrExternalId(
|
||||
prisma: Pick<PrismaClient, "session">,
|
||||
runtimeEnvironmentId: string,
|
||||
idOrExternalId: string
|
||||
): Promise<Session | null> {
|
||||
if (isSessionFriendlyIdForm(idOrExternalId)) {
|
||||
return prisma.session.findFirst({
|
||||
where: { friendlyId: idOrExternalId, runtimeEnvironmentId },
|
||||
});
|
||||
}
|
||||
|
||||
// `findFirst` rather than `findUnique` per the repo rule — `findUnique`'s
|
||||
// implicit DataLoader has open correctness bugs in Prisma 6.x that bite
|
||||
// hot-path lookups exactly like this one.
|
||||
return prisma.session.findFirst({
|
||||
where: { runtimeEnvironmentId, externalId: idOrExternalId },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Replica-first session resolution with a writer fallback on miss. For the
|
||||
* hot realtime routes (append / SSE subscribe / end-and-continue): a fresh
|
||||
* session's first append or subscribe can arrive inside the replica's apply
|
||||
* window, and a bare replica miss there surfaces as a 404 (or a subscribe
|
||||
* that never finds its session) for a session that exists on the writer.
|
||||
*/
|
||||
export async function resolveSessionWithWriterFallback(
|
||||
runtimeEnvironmentId: string,
|
||||
idOrExternalId: string
|
||||
): Promise<Session | null> {
|
||||
const row = await resolveSessionByIdOrExternalId($replica, runtimeEnvironmentId, idOrExternalId);
|
||||
if (row) return row;
|
||||
return resolveSessionByIdOrExternalId(prisma, runtimeEnvironmentId, idOrExternalId);
|
||||
}
|
||||
|
||||
/** True for `session_*` friendlyId form, false for everything else. */
|
||||
export function isSessionFriendlyIdForm(value: string): boolean {
|
||||
return value.startsWith(SESSION_FRIENDLY_ID_PREFIX);
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonicalise the addressing key used for everything stream-level: the
|
||||
* S2 stream path and the run-engine waitpoint cache key. `chat.agent`
|
||||
* and the rest of the operational surface always pass `externalId`, but
|
||||
* a public-API caller may legitimately address by `friendlyId` — and a
|
||||
* session created without an `externalId` only has a friendlyId at all.
|
||||
*
|
||||
* Rule:
|
||||
* - If we have a Session row, the canonical key is `externalId` if
|
||||
* set, else `friendlyId`. This way two callers addressing the same
|
||||
* row via different forms always converge to the same S2 stream.
|
||||
* - If we have no row (yet — chat.agent's transport may subscribe
|
||||
* before the agent's bind-time upsert lands), the canonical key is
|
||||
* whatever the URL had. Operationally that's always an externalId.
|
||||
* Friendlyid-form callers without a matching row are rejected by
|
||||
* the route handler before this is reached.
|
||||
*/
|
||||
export function canonicalSessionAddressingKey(row: Session | null, paramSession: string): string {
|
||||
if (row) {
|
||||
return row.externalId ?? row.friendlyId;
|
||||
}
|
||||
return paramSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Prisma `Session` row to the public {@link SessionItem} wire format.
|
||||
* Strips internal columns (project/environment/organization ids) and narrows
|
||||
* the `metadata` JSON to a record.
|
||||
*
|
||||
* Note: `currentRunId` is left as-is — Prisma stores the internal run id
|
||||
* (cuid), but `SessionItem.currentRunId` is the *friendly* form. Routes
|
||||
* that emit `SessionItem`s must translate: single-row endpoints via
|
||||
* {@link serializeSessionWithFriendlyRunId}, list endpoints via the
|
||||
* batched {@link serializeSessionsWithFriendlyRunIds}. Never put this
|
||||
* raw form on the wire directly.
|
||||
*/
|
||||
export function serializeSession(session: Session): SessionItem {
|
||||
return {
|
||||
id: session.friendlyId,
|
||||
externalId: session.externalId,
|
||||
type: session.type,
|
||||
taskIdentifier: session.taskIdentifier,
|
||||
triggerConfig: session.triggerConfig as SessionItem["triggerConfig"],
|
||||
currentRunId: session.currentRunId,
|
||||
tags: session.tags,
|
||||
metadata: (session.metadata ?? null) as SessionItem["metadata"],
|
||||
closedAt: session.closedAt,
|
||||
closedReason: session.closedReason,
|
||||
expiresAt: session.expiresAt,
|
||||
createdAt: session.createdAt,
|
||||
updatedAt: session.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as {@link serializeSession} but resolves `currentRunId` from the
|
||||
* internal cuid to the public `run_*` friendlyId via a TaskRun lookup.
|
||||
* Single-row endpoints (`POST/GET/PATCH/close /api/v1/sessions/:s`) use
|
||||
* this so the wire-side `currentRunId` is consistent with the rest of
|
||||
* the public API (which only accepts friendlyIds for run lookups).
|
||||
*
|
||||
* Skips the lookup when `currentRunId` is null.
|
||||
*
|
||||
* Resolves `currentRunId` -> `friendlyId` through `runStore.findRun` so a
|
||||
* run-ops id (NEW-DB) session run resolves from its owning store rather than the
|
||||
* control-plane replica. Mirrors `sessionRunManager.server.ts`.
|
||||
* Tenant-scoped because `Session.currentRunId` is a no-FK pointer.
|
||||
*/
|
||||
export async function serializeSessionWithFriendlyRunId(
|
||||
session: Session,
|
||||
runStore: RunStore = defaultRunStore
|
||||
): Promise<SessionItem> {
|
||||
const base = serializeSession(session);
|
||||
if (!session.currentRunId) return base;
|
||||
|
||||
const run = await runStore.findRun(
|
||||
{
|
||||
id: session.currentRunId,
|
||||
projectId: session.projectId,
|
||||
runtimeEnvironmentId: session.runtimeEnvironmentId,
|
||||
},
|
||||
{ select: { friendlyId: true } }
|
||||
);
|
||||
|
||||
return {
|
||||
...base,
|
||||
currentRunId: run?.friendlyId ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched form of {@link serializeSessionWithFriendlyRunId} for list
|
||||
* endpoints: one `IN` lookup per page instead of N+1. `currentRunId` on
|
||||
* the wire is always the public `run_*` friendlyId — the raw
|
||||
* {@link serializeSession} form leaks the internal cuid, which customers
|
||||
* can't use with `runs.retrieve(...)`.
|
||||
*/
|
||||
export async function serializeSessionsWithFriendlyRunIds(
|
||||
sessions: Session[],
|
||||
scope: { projectId: string; runtimeEnvironmentId: string },
|
||||
runStore: RunStore = defaultRunStore
|
||||
): Promise<SessionItem[]> {
|
||||
const runIds = [
|
||||
...new Set(sessions.map((s) => s.currentRunId).filter((id): id is string => !!id)),
|
||||
];
|
||||
|
||||
// `runStore.findRuns` fans out across both stores under split (NEW + LEGACY
|
||||
// replica merge) and is a plain `$replica` find when split is off. Tenant-
|
||||
// scoped: `Session.currentRunId` is a no-FK pointer, so a stale id must never
|
||||
// resolve a run in another env.
|
||||
const runs =
|
||||
runIds.length > 0
|
||||
? await runStore.findRuns({
|
||||
where: {
|
||||
id: { in: runIds },
|
||||
projectId: scope.projectId,
|
||||
runtimeEnvironmentId: scope.runtimeEnvironmentId,
|
||||
},
|
||||
select: { id: true, friendlyId: true },
|
||||
})
|
||||
: [];
|
||||
const friendlyIdByRunId = new Map(runs.map((run) => [run.id, run.friendlyId]));
|
||||
|
||||
return sessions.map((session) => ({
|
||||
...serializeSession(session),
|
||||
currentRunId: session.currentRunId
|
||||
? (friendlyIdByRunId.get(session.currentRunId) ?? null)
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import {
|
||||
type ElectricColumnType,
|
||||
RUN_ELECTRIC_COLUMNS,
|
||||
serializeRunRow,
|
||||
} from "./electricStreamProtocol.server";
|
||||
import { type RunHydrator, type RunListFilter, type RunListResolver } from "./runReader.server";
|
||||
|
||||
/**
|
||||
* Dual-run shadow-compare: the client is always served the Electric response while this re-derives what
|
||||
* the native backend would emit and diffs the two, to prove parity on real traffic before cutover. Checks
|
||||
* serialization (semantic per-column compare, gated on same updatedAt so a changed row is "skew", not a
|
||||
* divergence) and membership (emitted id-set, only on tag/batch initial snapshots). Pure but for the injected deps.
|
||||
*/
|
||||
|
||||
export type ShadowFeed = "run" | "runs" | "batch";
|
||||
|
||||
type WireValue = Record<string, string | null>;
|
||||
|
||||
type ShapeMessage = {
|
||||
key?: string;
|
||||
value?: WireValue;
|
||||
headers: { operation?: string; control?: string };
|
||||
};
|
||||
|
||||
const COLUMN_BY_NAME = new Map(RUN_ELECTRIC_COLUMNS.map((column) => [column.name, column]));
|
||||
|
||||
export type ColumnDiff = {
|
||||
runId: string;
|
||||
column: string;
|
||||
electric: string | null;
|
||||
native: string | null;
|
||||
};
|
||||
|
||||
export type ShadowCompareOutcome = {
|
||||
feed: ShadowFeed;
|
||||
/** Runs whose every emitted column matched (same-version). */
|
||||
serializationMatched: number;
|
||||
/** Runs with at least one semantic column divergence (same-version). */
|
||||
serializationDiverged: number;
|
||||
/** Runs that changed between Electric's emit and our refetch (not a divergence). */
|
||||
serializationSkew: number;
|
||||
/** Per-column divergences (capped) for logging. */
|
||||
diffs: ColumnDiff[];
|
||||
/** Set membership (tag/batch initial snapshot only). undefined when not checked. */
|
||||
membershipMatch?: boolean;
|
||||
missingInNative?: string[];
|
||||
extraInNative?: string[];
|
||||
};
|
||||
|
||||
export type ShadowCompareInput = {
|
||||
feed: ShadowFeed;
|
||||
/** The served Electric response body (a JSON array of messages, or "" / "[]"). */
|
||||
electricBody: string;
|
||||
environment: { id: string };
|
||||
skipColumns: string[];
|
||||
/** True when this was an initial snapshot request (offset=-1); enables membership compare. */
|
||||
isInitialSnapshot: boolean;
|
||||
/** When set (tag/batch initial snapshot), compare the resolved id-set. */
|
||||
membershipFilter?: RunListFilter;
|
||||
};
|
||||
|
||||
const MAX_DIFFS = 20;
|
||||
|
||||
export class RealtimeShadowComparator {
|
||||
constructor(
|
||||
private readonly options: { runReader: RunHydrator; runListResolver: RunListResolver }
|
||||
) {}
|
||||
|
||||
async compare(input: ShadowCompareInput): Promise<ShadowCompareOutcome> {
|
||||
const messages = parseBody(input.electricBody);
|
||||
const changes = messages.filter(
|
||||
(m): m is ShapeMessage & { value: WireValue } =>
|
||||
typeof m.headers?.operation === "string" && !!m.value && m.headers.operation !== "delete"
|
||||
);
|
||||
|
||||
const outcome: ShadowCompareOutcome = {
|
||||
feed: input.feed,
|
||||
serializationMatched: 0,
|
||||
serializationDiverged: 0,
|
||||
serializationSkew: 0,
|
||||
diffs: [],
|
||||
};
|
||||
|
||||
// Bulk-hydrate every emitted run in one query rather than a per-message round
|
||||
// trip, so shadow mode doesn't inflate the very replica load it's measuring.
|
||||
const emittedIds = changes
|
||||
.map((m) => m.value.id)
|
||||
.filter((id): id is string => typeof id === "string");
|
||||
const hydrated = await this.options.runReader.hydrateByIds(input.environment.id, emittedIds);
|
||||
const rowsById = new Map(hydrated.map((row) => [row.id, row]));
|
||||
|
||||
for (const message of changes) {
|
||||
const runId = message.value.id ?? undefined;
|
||||
if (!runId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const row = rowsById.get(runId);
|
||||
if (!row) {
|
||||
// Run no longer readable (deleted / replica miss). Not a serialization divergence.
|
||||
outcome.serializationSkew++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const nativeValue = serializeRunRow(row, input.skipColumns);
|
||||
|
||||
// Only compare rows at the same version; otherwise the row advanced between
|
||||
// Electric's emit and our refetch (timing skew, not a divergence).
|
||||
if (!sameInstant(message.value.updatedAt, nativeValue.updatedAt)) {
|
||||
outcome.serializationSkew++;
|
||||
continue;
|
||||
}
|
||||
|
||||
let rowDiverged = false;
|
||||
for (const [column, electricRaw] of Object.entries(message.value)) {
|
||||
const meta = COLUMN_BY_NAME.get(column);
|
||||
if (!meta) {
|
||||
continue;
|
||||
}
|
||||
const nativeRaw = nativeValue[column] ?? null;
|
||||
if (!valuesEqual(electricRaw, nativeRaw, meta.type, meta.dims, column)) {
|
||||
rowDiverged = true;
|
||||
if (outcome.diffs.length < MAX_DIFFS) {
|
||||
outcome.diffs.push({ runId, column, electric: electricRaw, native: nativeRaw });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rowDiverged) {
|
||||
outcome.serializationDiverged++;
|
||||
} else {
|
||||
outcome.serializationMatched++;
|
||||
}
|
||||
}
|
||||
|
||||
if (input.isInitialSnapshot && input.membershipFilter) {
|
||||
const electricIds = new Set(
|
||||
changes.map((m) => m.value.id).filter((id): id is string => typeof id === "string")
|
||||
);
|
||||
const nativeIds = new Set(
|
||||
await this.options.runListResolver.resolveMatchingRunIds(input.membershipFilter)
|
||||
);
|
||||
|
||||
outcome.missingInNative = [...electricIds].filter((id) => !nativeIds.has(id));
|
||||
outcome.extraInNative = [...nativeIds].filter((id) => !electricIds.has(id));
|
||||
outcome.membershipMatch =
|
||||
outcome.missingInNative.length === 0 && outcome.extraInNative.length === 0;
|
||||
}
|
||||
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
|
||||
function parseBody(body: string): ShapeMessage[] {
|
||||
const text = body.trim();
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
return Array.isArray(parsed) ? (parsed as ShapeMessage[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Status carries a known legacy rewrite (DEQUEUED -> EXECUTING) applied equally to
|
||||
* both paths for non-current API versions; treat them as equivalent. */
|
||||
function normalizeStatus(value: string): string {
|
||||
return value === "DEQUEUED" ? "EXECUTING" : value;
|
||||
}
|
||||
|
||||
function sameInstant(a: string | null | undefined, b: string | null | undefined): boolean {
|
||||
if (a == null || b == null) {
|
||||
return a == null && b == null;
|
||||
}
|
||||
// Mirror the SDK's RawShapeDate (`new Date(val + "Z")`).
|
||||
return new Date(`${a}Z`).getTime() === new Date(`${b}Z`).getTime();
|
||||
}
|
||||
|
||||
function valuesEqual(
|
||||
electricRaw: string | null,
|
||||
nativeRaw: string | null,
|
||||
type: ElectricColumnType,
|
||||
dims: number | undefined,
|
||||
column: string
|
||||
): boolean {
|
||||
if (electricRaw == null || nativeRaw == null) {
|
||||
return electricRaw == null && nativeRaw == null;
|
||||
}
|
||||
|
||||
if (dims && dims > 0) {
|
||||
return arraysEqual(parsePgTextArray(electricRaw), parsePgTextArray(nativeRaw));
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case "timestamp":
|
||||
return new Date(`${electricRaw}Z`).getTime() === new Date(`${nativeRaw}Z`).getTime();
|
||||
case "bool":
|
||||
return parseBool(electricRaw) === parseBool(nativeRaw);
|
||||
case "int4":
|
||||
case "int8":
|
||||
case "float8":
|
||||
return Number(electricRaw) === Number(nativeRaw);
|
||||
case "jsonb":
|
||||
return jsonEqual(electricRaw, nativeRaw);
|
||||
case "text":
|
||||
default:
|
||||
if (column === "status") {
|
||||
return normalizeStatus(electricRaw) === normalizeStatus(nativeRaw);
|
||||
}
|
||||
return electricRaw === nativeRaw;
|
||||
}
|
||||
}
|
||||
|
||||
function parseBool(value: string): boolean {
|
||||
return value === "t" || value === "true";
|
||||
}
|
||||
|
||||
function jsonEqual(a: string, b: string): boolean {
|
||||
try {
|
||||
return deepEqual(JSON.parse(a), JSON.parse(b));
|
||||
} catch {
|
||||
return a === b;
|
||||
}
|
||||
}
|
||||
|
||||
function deepEqual(a: unknown, b: unknown): boolean {
|
||||
if (a === b) return true;
|
||||
if (typeof a !== typeof b || a === null || b === null) return false;
|
||||
if (Array.isArray(a) && Array.isArray(b)) {
|
||||
return a.length === b.length && a.every((v, i) => deepEqual(v, b[i]));
|
||||
}
|
||||
if (typeof a === "object" && typeof b === "object") {
|
||||
const ak = Object.keys(a as object).sort();
|
||||
const bk = Object.keys(b as object).sort();
|
||||
return (
|
||||
ak.length === bk.length &&
|
||||
ak.every((k, i) => k === bk[i]) &&
|
||||
ak.every((k) => deepEqual((a as any)[k], (b as any)[k]))
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function arraysEqual(a: string[], b: string[]): boolean {
|
||||
return a.length === b.length && a.every((v, i) => v === b[i]);
|
||||
}
|
||||
|
||||
/** Parse a Postgres text-array literal (`{"a","b"}` / `{}`). Mirrors the client's pgArrayParser. */
|
||||
function parsePgTextArray(literal: string): string[] {
|
||||
if (literal === "{}" || literal === "") {
|
||||
return [];
|
||||
}
|
||||
const inner = literal.startsWith("{") && literal.endsWith("}") ? literal.slice(1, -1) : literal;
|
||||
const result: string[] = [];
|
||||
let i = 0;
|
||||
while (i < inner.length) {
|
||||
if (inner[i] === '"') {
|
||||
i++;
|
||||
let s = "";
|
||||
while (i < inner.length && inner[i] !== '"') {
|
||||
if (inner[i] === "\\") {
|
||||
i++;
|
||||
}
|
||||
s += inner[i];
|
||||
i++;
|
||||
}
|
||||
result.push(s);
|
||||
i++;
|
||||
if (inner[i] === ",") i++;
|
||||
} else {
|
||||
let s = "";
|
||||
while (i < inner.length && inner[i] !== ",") {
|
||||
s += inner[i];
|
||||
i++;
|
||||
}
|
||||
result.push(s);
|
||||
if (inner[i] === ",") i++;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
import type { API_VERSIONS } from "~/api/versions";
|
||||
import { logger } from "../logger.server";
|
||||
import {
|
||||
type RealtimeEnvironment,
|
||||
type RealtimeRequestOptions,
|
||||
type RealtimeRunsParams,
|
||||
} from "../realtimeClient.server";
|
||||
import { RESERVED_COLUMNS } from "./electricStreamProtocol.server";
|
||||
import {
|
||||
type RealtimeListEnvironment,
|
||||
type RealtimeStreamClient,
|
||||
} from "./nativeRealtimeClient.server";
|
||||
import { type RunListFilter } from "./runReader.server";
|
||||
import {
|
||||
type RealtimeShadowComparator,
|
||||
type ShadowCompareOutcome,
|
||||
type ShadowFeed,
|
||||
} from "./shadowCompare.server";
|
||||
|
||||
export type ShadowRealtimeClientOptions = {
|
||||
/** The path actually served to the client (Electric). */
|
||||
electric: RealtimeStreamClient;
|
||||
comparator: RealtimeShadowComparator;
|
||||
/** createdAt window (ms) used to resolve tag-list membership for the compare. */
|
||||
maximumCreatedAtFilterAgeMs: number;
|
||||
/** Cap for the membership resolve. */
|
||||
maxListResults: number;
|
||||
/** Metrics sink for compare outcomes. */
|
||||
onOutcome?: (outcome: ShadowCompareOutcome) => void;
|
||||
};
|
||||
|
||||
/** Transparent wrapper that serves the Electric response unchanged and, in the background (fire-and-forget), diffs what the native backend would emit. */
|
||||
export class ShadowRealtimeClient implements RealtimeStreamClient {
|
||||
constructor(private readonly options: ShadowRealtimeClientOptions) {}
|
||||
|
||||
async streamRun(
|
||||
url: URL | string,
|
||||
environment: RealtimeEnvironment,
|
||||
runId: string,
|
||||
apiVersion: API_VERSIONS,
|
||||
requestOptions?: RealtimeRequestOptions,
|
||||
clientVersion?: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Response> {
|
||||
const response = await this.options.electric.streamRun(
|
||||
url,
|
||||
environment,
|
||||
runId,
|
||||
apiVersion,
|
||||
requestOptions,
|
||||
clientVersion,
|
||||
signal
|
||||
);
|
||||
this.#shadow("run", response, url, environment, requestOptions);
|
||||
return response;
|
||||
}
|
||||
|
||||
async streamRuns(
|
||||
url: URL | string,
|
||||
environment: RealtimeListEnvironment,
|
||||
params: RealtimeRunsParams,
|
||||
apiVersion: API_VERSIONS,
|
||||
requestOptions?: RealtimeRequestOptions,
|
||||
clientVersion?: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Response> {
|
||||
const response = await this.options.electric.streamRuns(
|
||||
url,
|
||||
environment,
|
||||
params,
|
||||
apiVersion,
|
||||
requestOptions,
|
||||
clientVersion,
|
||||
signal
|
||||
);
|
||||
this.#shadow("runs", response, url, environment, requestOptions, { tags: params.tags ?? [] });
|
||||
return response;
|
||||
}
|
||||
|
||||
async streamBatch(
|
||||
url: URL | string,
|
||||
environment: RealtimeListEnvironment,
|
||||
batchId: string,
|
||||
apiVersion: API_VERSIONS,
|
||||
requestOptions?: RealtimeRequestOptions,
|
||||
clientVersion?: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Response> {
|
||||
const response = await this.options.electric.streamBatch(
|
||||
url,
|
||||
environment,
|
||||
batchId,
|
||||
apiVersion,
|
||||
requestOptions,
|
||||
clientVersion,
|
||||
signal
|
||||
);
|
||||
this.#shadow("batch", response, url, environment, requestOptions, { batchId });
|
||||
return response;
|
||||
}
|
||||
|
||||
/** Fire-and-forget; never blocks the served response, never throws into the request. */
|
||||
#shadow(
|
||||
feed: ShadowFeed,
|
||||
electricResponse: Response,
|
||||
url: URL | string,
|
||||
environment: RealtimeEnvironment & { projectId?: string },
|
||||
requestOptions?: RealtimeRequestOptions,
|
||||
membership?: { tags?: string[]; batchId?: string }
|
||||
): void {
|
||||
// Clone synchronously before the client consumes the body.
|
||||
let bodyClone: Response;
|
||||
try {
|
||||
if (electricResponse.status !== 200) {
|
||||
return;
|
||||
}
|
||||
bodyClone = electricResponse.clone();
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
void this.#runShadow(feed, bodyClone, url, environment, requestOptions, membership).catch(
|
||||
(error) => logger.debug("[shadowRealtime] compare failed", { feed, error })
|
||||
);
|
||||
}
|
||||
|
||||
async #runShadow(
|
||||
feed: ShadowFeed,
|
||||
bodyClone: Response,
|
||||
url: URL | string,
|
||||
environment: RealtimeEnvironment & { projectId?: string },
|
||||
requestOptions: RealtimeRequestOptions | undefined,
|
||||
membership: { tags?: string[]; batchId?: string } | undefined
|
||||
): Promise<void> {
|
||||
const $url = new URL(url.toString());
|
||||
const offset = $url.searchParams.get("offset") ?? "-1";
|
||||
const handle = $url.searchParams.get("handle") ?? $url.searchParams.get("shape_id");
|
||||
const isInitialSnapshot = offset === "-1" || !handle;
|
||||
const skipColumns = resolveSkipColumns($url, requestOptions);
|
||||
const electricBody = await bodyClone.text();
|
||||
|
||||
let membershipFilter: RunListFilter | undefined;
|
||||
if (isInitialSnapshot && membership && environment.projectId) {
|
||||
membershipFilter = {
|
||||
organizationId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
tags: membership.tags,
|
||||
batchId: membership.batchId,
|
||||
createdAtAfter: membership.batchId
|
||||
? undefined
|
||||
: new Date(Date.now() - this.options.maximumCreatedAtFilterAgeMs),
|
||||
limit: this.options.maxListResults,
|
||||
};
|
||||
}
|
||||
|
||||
const outcome = await this.options.comparator.compare({
|
||||
feed,
|
||||
electricBody,
|
||||
environment: { id: environment.id },
|
||||
skipColumns,
|
||||
isInitialSnapshot,
|
||||
membershipFilter,
|
||||
});
|
||||
|
||||
this.options.onOutcome?.(outcome);
|
||||
|
||||
if (outcome.serializationDiverged > 0 || outcome.membershipMatch === false) {
|
||||
logger.warn("[shadowRealtime] divergence detected", {
|
||||
feed,
|
||||
serializationDiverged: outcome.serializationDiverged,
|
||||
serializationMatched: outcome.serializationMatched,
|
||||
serializationSkew: outcome.serializationSkew,
|
||||
membershipMatch: outcome.membershipMatch,
|
||||
missingInNative: outcome.missingInNative?.slice(0, 20),
|
||||
extraInNative: outcome.extraInNative?.slice(0, 20),
|
||||
// Log only which run/column diverged, never the raw cell values — they can
|
||||
// include run payload/output/metadata and must not leak into logs.
|
||||
diffs: outcome.diffs.map(({ runId, column }) => ({ runId, column })),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSkipColumns(url: URL, requestOptions?: RealtimeRequestOptions): string[] {
|
||||
const raw = requestOptions?.skipColumns ?? url.searchParams.get("skipColumns")?.split(",") ?? [];
|
||||
return raw.map((c) => c.trim()).filter((c) => c !== "" && !RESERVED_COLUMNS.includes(c));
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
import { $replica } from "~/db.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { env } from "~/env.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { realtimeClient } from "../realtimeClientGlobal.server";
|
||||
import { ClickHouseRunListResolver } from "./clickHouseRunListResolver.server";
|
||||
import { RunHydrator } from "./runReader.server";
|
||||
import { RealtimeShadowComparator } from "./shadowCompare.server";
|
||||
import { ShadowRealtimeClient } from "./shadowRealtimeClient.server";
|
||||
|
||||
/**
|
||||
* Process-singleton wiring for the shadow-compare client. Only constructed
|
||||
* when an org's `realtimeBackend` flag is set to "shadow".
|
||||
*/
|
||||
function initializeShadowRealtimeClient(): ShadowRealtimeClient {
|
||||
const compares = getMeter("realtime-shadow").createCounter("realtime_shadow.compares", {
|
||||
description:
|
||||
"Dual-run shadow-compare outcomes (Electric vs native). kind=serialization|membership, result=match|diverge|skew.",
|
||||
});
|
||||
|
||||
const comparator = new RealtimeShadowComparator({
|
||||
runReader: new RunHydrator({ replica: $replica, runStore }),
|
||||
runListResolver: new ClickHouseRunListResolver({
|
||||
getClickhouse: (organizationId) =>
|
||||
clickhouseFactory.getClickhouseForOrganization(organizationId, "realtime"),
|
||||
prisma: $replica,
|
||||
}),
|
||||
});
|
||||
|
||||
return new ShadowRealtimeClient({
|
||||
electric: realtimeClient,
|
||||
comparator,
|
||||
maximumCreatedAtFilterAgeMs: env.REALTIME_MAXIMUM_CREATED_AT_FILTER_AGE_IN_MS,
|
||||
maxListResults: env.REALTIME_BACKEND_NATIVE_MAX_LIST_RESULTS,
|
||||
onOutcome: (outcome) => {
|
||||
const { feed } = outcome;
|
||||
if (outcome.serializationMatched) {
|
||||
compares.add(outcome.serializationMatched, {
|
||||
feed,
|
||||
kind: "serialization",
|
||||
result: "match",
|
||||
});
|
||||
}
|
||||
if (outcome.serializationDiverged) {
|
||||
compares.add(outcome.serializationDiverged, {
|
||||
feed,
|
||||
kind: "serialization",
|
||||
result: "diverge",
|
||||
});
|
||||
}
|
||||
if (outcome.serializationSkew) {
|
||||
compares.add(outcome.serializationSkew, { feed, kind: "serialization", result: "skew" });
|
||||
}
|
||||
if (outcome.membershipMatch !== undefined) {
|
||||
compares.add(1, {
|
||||
feed,
|
||||
kind: "membership",
|
||||
result: outcome.membershipMatch ? "match" : "diverge",
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function getShadowRealtimeClient(): ShadowRealtimeClient {
|
||||
return singleton("shadowRealtimeClient", initializeShadowRealtimeClient);
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Per-org S2 basin provisioning. Gated by
|
||||
* `REALTIME_STREAMS_PER_ORG_BASINS_ENABLED`: when off, all orgs share
|
||||
* `REALTIME_STREAMS_S2_BASIN` and this module no-ops.
|
||||
*
|
||||
* Pure retention-string in / S2-call out. Plan vocabulary lives in the
|
||||
* cloud billing app, which calls into the admin sync route to drive
|
||||
* provisioning + reconfiguration.
|
||||
*/
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { parseDuration } from "./duration.server";
|
||||
|
||||
export function isPerOrgBasinsEnabled(): boolean {
|
||||
return env.REALTIME_STREAMS_PER_ORG_BASINS_ENABLED === "true";
|
||||
}
|
||||
|
||||
export function defaultRetention(): string {
|
||||
return env.REALTIME_STREAMS_BASIN_DEFAULT_RETENTION;
|
||||
}
|
||||
|
||||
// Org id is a cuid — fixed-length and stable, so the basin name is
|
||||
// collision-free without truncation. Slugs are user-editable and would
|
||||
// drift.
|
||||
export function basinNameForOrg(org: { id: string }): string {
|
||||
const prefix = env.REALTIME_STREAMS_BASIN_NAME_PREFIX;
|
||||
const envName = env.REALTIME_STREAMS_BASIN_NAME_ENV;
|
||||
return `${prefix}-${envName}-org-${org.id}`;
|
||||
}
|
||||
|
||||
type ProvisionInput = {
|
||||
id: string;
|
||||
retention?: string;
|
||||
streamBasinName: string | null | undefined;
|
||||
};
|
||||
|
||||
type ProvisionResult =
|
||||
| { kind: "skipped"; reason: "feature-disabled" | "already-provisioned"; basin: string | null }
|
||||
| { kind: "provisioned"; basin: string; retention: string };
|
||||
|
||||
// Idempotent. Treats S2 409 as success (race with another caller, or
|
||||
// previous run that crashed after S2 ack but before the column write).
|
||||
export async function provisionBasinForOrg(
|
||||
org: ProvisionInput,
|
||||
prismaClient: PrismaClientOrTransaction = prisma
|
||||
): Promise<ProvisionResult> {
|
||||
if (!isPerOrgBasinsEnabled()) {
|
||||
return { kind: "skipped", reason: "feature-disabled", basin: null };
|
||||
}
|
||||
|
||||
if (org.streamBasinName) {
|
||||
return { kind: "skipped", reason: "already-provisioned", basin: org.streamBasinName };
|
||||
}
|
||||
|
||||
const accessToken = env.REALTIME_STREAMS_S2_ACCESS_TOKEN;
|
||||
if (!accessToken) {
|
||||
throw new Error(
|
||||
"REALTIME_STREAMS_S2_ACCESS_TOKEN must be set when REALTIME_STREAMS_PER_ORG_BASINS_ENABLED=true"
|
||||
);
|
||||
}
|
||||
|
||||
const basin = basinNameForOrg(org);
|
||||
const retention = org.retention ?? defaultRetention();
|
||||
|
||||
await s2CreateBasin(basin, {
|
||||
accessToken,
|
||||
retentionPolicy: retention,
|
||||
storageClass: env.REALTIME_STREAMS_BASIN_STORAGE_CLASS,
|
||||
deleteOnEmptyMinAge: env.REALTIME_STREAMS_BASIN_DELETE_ON_EMPTY_MIN_AGE,
|
||||
});
|
||||
|
||||
await prismaClient.organization.update({
|
||||
where: { id: org.id },
|
||||
data: { streamBasinName: basin },
|
||||
});
|
||||
|
||||
// streamBasinName is embedded in every env of the org; drop all its cached env rows.
|
||||
controlPlaneResolver.invalidateOrganization(org.id);
|
||||
|
||||
logger.info("[streamBasinProvisioner] provisioned basin for org", {
|
||||
orgId: org.id,
|
||||
basin,
|
||||
retention,
|
||||
});
|
||||
|
||||
return { kind: "provisioned", basin, retention };
|
||||
}
|
||||
|
||||
export async function reconfigureBasinForOrg(orgId: string, retention: string): Promise<void> {
|
||||
if (!isPerOrgBasinsEnabled()) return;
|
||||
|
||||
const accessToken = env.REALTIME_STREAMS_S2_ACCESS_TOKEN;
|
||||
if (!accessToken) {
|
||||
throw new Error(
|
||||
"REALTIME_STREAMS_S2_ACCESS_TOKEN must be set when REALTIME_STREAMS_PER_ORG_BASINS_ENABLED=true"
|
||||
);
|
||||
}
|
||||
|
||||
const org = await prisma.organization.findFirst({
|
||||
where: { id: orgId },
|
||||
select: { id: true, streamBasinName: true },
|
||||
});
|
||||
if (!org?.streamBasinName) return;
|
||||
|
||||
await s2ReconfigureBasin(org.streamBasinName, { accessToken, retentionPolicy: retention });
|
||||
|
||||
logger.info("[streamBasinProvisioner] reconfigured basin retention", {
|
||||
orgId,
|
||||
basin: org.streamBasinName,
|
||||
retention,
|
||||
});
|
||||
}
|
||||
|
||||
type EnsureResult =
|
||||
| { kind: "skipped"; reason: "feature-disabled" | "org-not-found" }
|
||||
| { kind: "provisioned"; basin: string; retention: string }
|
||||
| { kind: "reconfigured"; basin: string; retention: string };
|
||||
|
||||
// Idempotent: provisions if the org has no basin, PATCHes retention if
|
||||
// it does. The single entrypoint the cloud billing app drives — both
|
||||
// for the live plan-change path and the bulk backfill.
|
||||
export async function ensureBasinForOrg(orgId: string, retention: string): Promise<EnsureResult> {
|
||||
if (!isPerOrgBasinsEnabled()) {
|
||||
return { kind: "skipped", reason: "feature-disabled" };
|
||||
}
|
||||
|
||||
const org = await prisma.organization.findFirst({
|
||||
where: { id: orgId },
|
||||
select: { id: true, streamBasinName: true },
|
||||
});
|
||||
if (!org) return { kind: "skipped", reason: "org-not-found" };
|
||||
|
||||
if (!org.streamBasinName) {
|
||||
const result = await provisionBasinForOrg({ id: org.id, streamBasinName: null, retention });
|
||||
if (result.kind === "provisioned") {
|
||||
return { kind: "provisioned", basin: result.basin, retention: result.retention };
|
||||
}
|
||||
return { kind: "skipped", reason: "feature-disabled" };
|
||||
}
|
||||
|
||||
await reconfigureBasinForOrg(org.id, retention);
|
||||
return { kind: "reconfigured", basin: org.streamBasinName, retention };
|
||||
}
|
||||
|
||||
// Inverse of ensureBasinForOrg: nulls the column so future runs/sessions
|
||||
// land in the shared global basin. The S2 basin lingers; existing streams
|
||||
// age out on their original retention.
|
||||
export async function deprovisionBasinForOrg(
|
||||
orgId: string
|
||||
): Promise<{ kind: "deprovisioned" } | { kind: "skipped"; reason: "no-basin" }> {
|
||||
const org = await prisma.organization.findFirst({
|
||||
where: { id: orgId },
|
||||
select: { id: true, streamBasinName: true },
|
||||
});
|
||||
if (!org?.streamBasinName) return { kind: "skipped", reason: "no-basin" };
|
||||
|
||||
await prisma.organization.update({
|
||||
where: { id: org.id },
|
||||
data: { streamBasinName: null },
|
||||
});
|
||||
|
||||
// streamBasinName is embedded in every env of the org; drop all its cached env rows.
|
||||
controlPlaneResolver.invalidateOrganization(org.id);
|
||||
|
||||
logger.info("[streamBasinProvisioner] deprovisioned basin for org", {
|
||||
orgId,
|
||||
previousBasin: org.streamBasinName,
|
||||
});
|
||||
|
||||
return { kind: "deprovisioned" };
|
||||
}
|
||||
|
||||
// S2 REST: POST /v1/basins to create, PATCH /v1/basins/{name} to
|
||||
// reconfigure. Wire shape takes integer seconds; we accept human strings
|
||||
// like "7d" / "1y" as env-var ergonomics and parse them here.
|
||||
|
||||
type CreateBasinOptions = {
|
||||
accessToken: string;
|
||||
retentionPolicy: string;
|
||||
storageClass: "express" | "standard";
|
||||
deleteOnEmptyMinAge: string;
|
||||
};
|
||||
|
||||
async function s2CreateBasin(name: string, opts: CreateBasinOptions): Promise<void> {
|
||||
const url = `https://aws.s2.dev/v1/basins`;
|
||||
const body = {
|
||||
basin: name,
|
||||
config: {
|
||||
create_stream_on_append: true,
|
||||
create_stream_on_read: true,
|
||||
default_stream_config: {
|
||||
storage_class: opts.storageClass,
|
||||
retention_policy: { age: parseDuration(opts.retentionPolicy) },
|
||||
delete_on_empty: { min_age_secs: parseDuration(opts.deleteOnEmptyMinAge) },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${opts.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
// 409 = basin already exists; treat as success (idempotent).
|
||||
if (res.ok || res.status === 409) return;
|
||||
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`S2 createBasin failed: ${res.status} ${res.statusText} ${text}`);
|
||||
}
|
||||
|
||||
type ReconfigureBasinOptions = {
|
||||
accessToken: string;
|
||||
retentionPolicy: string;
|
||||
};
|
||||
|
||||
async function s2ReconfigureBasin(name: string, opts: ReconfigureBasinOptions): Promise<void> {
|
||||
const url = `https://aws.s2.dev/v1/basins/${encodeURIComponent(name)}`;
|
||||
const body = {
|
||||
default_stream_config: {
|
||||
retention_policy: { age: parseDuration(opts.retentionPolicy) },
|
||||
},
|
||||
};
|
||||
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
Authorization: `Bearer ${opts.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (res.ok) return;
|
||||
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`S2 reconfigureBasin failed: ${res.status} ${res.statusText} ${text}`);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
export type StreamRecord = {
|
||||
data: string;
|
||||
id: string;
|
||||
seqNum: number;
|
||||
/**
|
||||
* S2 record headers, when the underlying backend is the v2 (S2) shape.
|
||||
* Undefined or empty for run-scoped Redis streams. First-header empty-name
|
||||
* is an S2 command record (trim/fence); the parser strips those before
|
||||
* surfacing the record, so callers never see them.
|
||||
*/
|
||||
headers?: Array<[string, string]>;
|
||||
};
|
||||
|
||||
// Interface for stream ingestion
|
||||
export interface StreamIngestor {
|
||||
initializeStream(
|
||||
runId: string,
|
||||
streamId: string
|
||||
): Promise<{ responseHeaders?: Record<string, string> }>;
|
||||
|
||||
ingestData(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
clientId: string,
|
||||
resumeFromChunk?: number
|
||||
): Promise<Response>;
|
||||
|
||||
appendPart(part: string, partId: string, runId: string, streamId: string): Promise<void>;
|
||||
|
||||
getLastChunkIndex(runId: string, streamId: string, clientId: string): Promise<number>;
|
||||
|
||||
readRecords(runId: string, streamId: string, afterSeqNum?: number): Promise<StreamRecord[]>;
|
||||
}
|
||||
|
||||
export type StreamResponseOptions = {
|
||||
timeoutInSeconds?: number;
|
||||
lastEventId?: string;
|
||||
/**
|
||||
* Session-stream-only. When `true`, the responder MAY peek the tail
|
||||
* of `.out` and short-circuit to `wait=0` + `X-Session-Settled: true`
|
||||
* if the last record is a terminal marker (a `trigger-control`
|
||||
* `turn-complete` control record, ignoring any trailing S2 trim
|
||||
* command record). Used by `TriggerChatTransport.reconnectToStream`
|
||||
* on page reload.
|
||||
*
|
||||
* When absent/false, the responder keeps the unconditional long-poll
|
||||
* behavior — required on the active send-a-message path where the
|
||||
* peek would race the newly-triggered turn's first chunk.
|
||||
*/
|
||||
peekSettled?: boolean;
|
||||
};
|
||||
|
||||
// Interface for stream response
|
||||
export interface StreamResponder {
|
||||
streamResponse(
|
||||
request: Request,
|
||||
runId: string,
|
||||
streamId: string,
|
||||
signal: AbortSignal,
|
||||
options?: StreamResponseOptions
|
||||
): Promise<Response>;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
export class LineTransformStream extends TransformStream<string, string[]> {
|
||||
private buffer = "";
|
||||
|
||||
constructor() {
|
||||
super({
|
||||
transform: (chunk, controller) => {
|
||||
// Append the chunk to the buffer
|
||||
this.buffer += chunk;
|
||||
|
||||
// Split on newlines
|
||||
const lines = this.buffer.split("\n");
|
||||
|
||||
// The last element might be incomplete, hold it back in buffer
|
||||
this.buffer = lines.pop() || "";
|
||||
|
||||
// Filter out empty or whitespace-only lines
|
||||
const fullLines = lines.filter((line) => line.trim().length > 0);
|
||||
|
||||
// If we got any complete lines, emit them as an array
|
||||
if (fullLines.length > 0) {
|
||||
controller.enqueue(fullLines);
|
||||
}
|
||||
},
|
||||
flush: (controller) => {
|
||||
// On stream end, if there's leftover text, emit it as a single-element array
|
||||
const trimmed = this.buffer.trim();
|
||||
if (trimmed.length > 0) {
|
||||
controller.enqueue([trimmed]);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import {
|
||||
createCache,
|
||||
createLRUMemoryStore,
|
||||
DefaultStatefulContext,
|
||||
Namespace,
|
||||
RedisCacheStore,
|
||||
} from "@internal/cache";
|
||||
import { env } from "~/env.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import type { AuthenticatedEnvironment } from "../apiAuth.server";
|
||||
import { RedisRealtimeStreams } from "./redisRealtimeStreams.server";
|
||||
import { S2RealtimeStreams } from "./s2realtimeStreams.server";
|
||||
import type { StreamIngestor, StreamResponder } from "./types";
|
||||
|
||||
function initializeRedisRealtimeStreams() {
|
||||
return new RedisRealtimeStreams({
|
||||
redis: {
|
||||
port: env.REALTIME_STREAMS_REDIS_PORT,
|
||||
host: env.REALTIME_STREAMS_REDIS_HOST,
|
||||
username: env.REALTIME_STREAMS_REDIS_USERNAME,
|
||||
password: env.REALTIME_STREAMS_REDIS_PASSWORD,
|
||||
enableAutoPipelining: true,
|
||||
...(env.REALTIME_STREAMS_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
keyPrefix: "tr:realtime:streams:",
|
||||
},
|
||||
inactivityTimeoutMs: env.REALTIME_STREAMS_INACTIVITY_TIMEOUT_MS,
|
||||
});
|
||||
}
|
||||
|
||||
export const v1RealtimeStreams = singleton("realtimeStreams", initializeRedisRealtimeStreams);
|
||||
|
||||
/**
|
||||
* Resolve a stream's basin. Precedence: run → session → org → global env.
|
||||
* Pre-migration rows have `streamBasinName: null` and fall through to
|
||||
* the global basin (where their streams actually live), so only pass
|
||||
* `organization` when no run/session row exists at all — otherwise a
|
||||
* null column would short-circuit to the org's *current* basin.
|
||||
*/
|
||||
export type StreamBasinContext = {
|
||||
run?: { streamBasinName: string | null } | null;
|
||||
session?: { streamBasinName: string | null } | null;
|
||||
organization?: { streamBasinName: string | null } | null;
|
||||
};
|
||||
|
||||
export function resolveStreamBasin(ctx: StreamBasinContext): string | undefined {
|
||||
return (
|
||||
ctx.run?.streamBasinName ??
|
||||
ctx.session?.streamBasinName ??
|
||||
ctx.organization?.streamBasinName ??
|
||||
env.REALTIME_STREAMS_S2_BASIN ??
|
||||
undefined
|
||||
);
|
||||
}
|
||||
|
||||
export function getRealtimeStreamInstance(
|
||||
environment: AuthenticatedEnvironment,
|
||||
streamVersion: string,
|
||||
basinContext?: StreamBasinContext
|
||||
): StreamIngestor & StreamResponder {
|
||||
if (streamVersion === "v1") {
|
||||
return v1RealtimeStreams;
|
||||
}
|
||||
|
||||
const resolvedBasin = resolveStreamBasin(basinContext ?? {});
|
||||
if (
|
||||
resolvedBasin &&
|
||||
(env.REALTIME_STREAMS_S2_ACCESS_TOKEN || env.REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS === "true")
|
||||
) {
|
||||
return new S2RealtimeStreams({
|
||||
basin: resolvedBasin,
|
||||
accessToken: env.REALTIME_STREAMS_S2_ACCESS_TOKEN ?? "",
|
||||
endpoint: env.REALTIME_STREAMS_S2_ENDPOINT,
|
||||
skipAccessTokens: env.REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS === "true",
|
||||
streamPrefix: streamPrefixFor(environment, resolvedBasin),
|
||||
logLevel: env.REALTIME_STREAMS_S2_LOG_LEVEL,
|
||||
flushIntervalMs: env.REALTIME_STREAMS_S2_FLUSH_INTERVAL_MS,
|
||||
maxRetries: env.REALTIME_STREAMS_S2_MAX_RETRIES,
|
||||
s2WaitSeconds: env.REALTIME_STREAMS_S2_WAIT_SECONDS,
|
||||
accessTokenExpirationInMs: env.REALTIME_STREAMS_S2_ACCESS_TOKEN_EXPIRATION_IN_MS,
|
||||
cache: s2RealtimeStreamsCache,
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error("Realtime streams v2 is required for this run but S2 configuration is missing");
|
||||
}
|
||||
|
||||
// Shared basin needs `org/{orgId}` to namespace; per-org basin already
|
||||
// isolates so the segment drops.
|
||||
function streamPrefixFor(environment: AuthenticatedEnvironment, basin: string): string {
|
||||
const isPerOrgBasin = basin !== env.REALTIME_STREAMS_S2_BASIN;
|
||||
const segments = isPerOrgBasin
|
||||
? ["env", environment.slug, environment.id]
|
||||
: ["org", environment.organization.id, "env", environment.slug, environment.id];
|
||||
return segments.join("/");
|
||||
}
|
||||
|
||||
export function determineRealtimeStreamsVersion(streamVersion?: string): "v1" | "v2" {
|
||||
if (!streamVersion) {
|
||||
return env.REALTIME_STREAMS_DEFAULT_VERSION;
|
||||
}
|
||||
|
||||
if (
|
||||
streamVersion === "v2" &&
|
||||
env.REALTIME_STREAMS_S2_BASIN &&
|
||||
(env.REALTIME_STREAMS_S2_ACCESS_TOKEN || env.REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS === "true")
|
||||
) {
|
||||
return "v2";
|
||||
}
|
||||
|
||||
return "v1";
|
||||
}
|
||||
|
||||
const s2RealtimeStreamsCache = singleton(
|
||||
"s2RealtimeStreamsCache",
|
||||
initializeS2RealtimeStreamsCache
|
||||
);
|
||||
|
||||
function initializeS2RealtimeStreamsCache() {
|
||||
const ctx = new DefaultStatefulContext();
|
||||
const redisCacheStore = new RedisCacheStore({
|
||||
name: "s2-realtime-streams-cache",
|
||||
connection: {
|
||||
port: env.REALTIME_STREAMS_REDIS_PORT,
|
||||
host: env.REALTIME_STREAMS_REDIS_HOST,
|
||||
username: env.REALTIME_STREAMS_REDIS_USERNAME,
|
||||
password: env.REALTIME_STREAMS_REDIS_PASSWORD,
|
||||
enableAutoPipelining: true,
|
||||
...(env.REALTIME_STREAMS_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
keyPrefix: "s2-realtime-streams-cache:",
|
||||
},
|
||||
useModernCacheKeyBuilder: true,
|
||||
});
|
||||
|
||||
const memoryStore = createLRUMemoryStore(5000);
|
||||
|
||||
return createCache({
|
||||
accessToken: new Namespace<string>(ctx, {
|
||||
stores: [memoryStore, redisCacheStore],
|
||||
fresh: Math.floor(env.REALTIME_STREAMS_S2_ACCESS_TOKEN_EXPIRATION_IN_MS / 2),
|
||||
stale: Math.floor(env.REALTIME_STREAMS_S2_ACCESS_TOKEN_EXPIRATION_IN_MS / 2 + 60_000),
|
||||
}),
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user