chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,206 @@
|
||||
import { applyMetadataOperations } from "@trigger.dev/core/v3";
|
||||
import type { FlushedRunMetadata } from "@trigger.dev/core/v3/schemas";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
|
||||
// On `applied` we surface the parent/root friendlyIds captured during
|
||||
// the snapshot read. Callers that fan parent/root metadata operations
|
||||
// out to their respective runs can use these without a second
|
||||
// `findRunByIdWithMollifierFallback` round trip — and, more importantly,
|
||||
// without racing the drainer's terminal-failure path (which atomically
|
||||
// DELetes the entry hash). Without these on the outcome the second
|
||||
// read can come back null mid-route, silently dropping the caller's
|
||||
// parentOperations / rootOperations after the primary mutation already
|
||||
// landed on the snapshot.
|
||||
//
|
||||
// FriendlyIds (not internal cuids) because the consuming
|
||||
// `routeOperationsToRun` helper gates on the `run_…` prefix to decide
|
||||
// whether to attempt the buffer fallback; cuids would skip that path.
|
||||
// The snapshot's `parentTaskRunId` / `rootTaskRunId` are engine-side
|
||||
// cuids, so we convert via `RunId.toFriendlyId` here — identical to
|
||||
// what `readFallback.server.ts` does when assembling its SyntheticRun.
|
||||
export type ApplyMetadataMutationOutcome =
|
||||
| {
|
||||
kind: "applied";
|
||||
newMetadata: Record<string, unknown>;
|
||||
parentTaskRunFriendlyId: string | undefined;
|
||||
rootTaskRunFriendlyId: string | undefined;
|
||||
}
|
||||
| { kind: "not_found" }
|
||||
| { kind: "busy" }
|
||||
| { kind: "version_exhausted" }
|
||||
// Mirrors the PG-side `MetadataTooLargeError` (status 413). Carries
|
||||
// the limit + observed size so the route can produce a useful body.
|
||||
| { kind: "metadata_too_large"; maximumSize: number; observedSize: number };
|
||||
|
||||
// Apply a metadata PUT (body.metadata replace AND/OR body.operations
|
||||
// deltas) to a buffered run's snapshot. Mirrors the PG-side
|
||||
// `UpdateMetadataService.#updateRunMetadataWithOperations` retry loop:
|
||||
// read snapshot → apply operations in JS → CAS-write back with the
|
||||
// observed `metadataVersion`. Retries on conflict; bounded by
|
||||
// `maxRetries`. The Lua CAS is the atomicity primitive — concurrent
|
||||
// callers never lose an increment / append / set.
|
||||
export async function applyMetadataMutationToBufferedRun(input: {
|
||||
runId: string;
|
||||
// Env+org scoping closes a cross-environment write gap on the buffer
|
||||
// path: the route's PG path is already env-scoped via Prisma filters,
|
||||
// and this helper now enforces the same isolation before any buffer
|
||||
// write so a caller authed in env A can't mutate a buffered run that
|
||||
// belongs to env B.
|
||||
environmentId: string;
|
||||
organizationId: string;
|
||||
// Byte-size cap on the resulting metadata payload, mirroring the
|
||||
// PG-side `UpdateMetadataService.maximumSize` (sourced from
|
||||
// `env.TASK_RUN_METADATA_MAXIMUM_SIZE`). Required so the buffer path
|
||||
// doesn't silently allow writes the PG path would have rejected.
|
||||
maximumSize: number;
|
||||
body: Pick<FlushedRunMetadata, "metadata" | "operations">;
|
||||
buffer?: MollifierBuffer | null;
|
||||
maxRetries?: number;
|
||||
// Jittered conflict-backoff envelope: random in [0, base + attempt * step) ms.
|
||||
backoffBaseMs?: number;
|
||||
backoffStepMs?: number;
|
||||
}): Promise<ApplyMetadataMutationOutcome> {
|
||||
const buffer = input.buffer ?? getMollifierBuffer();
|
||||
if (!buffer) return { kind: "not_found" };
|
||||
|
||||
// Default retry budget tuned for buffered-window concurrency. The
|
||||
// PG-side `UpdateMetadataService` uses 3, which is fine when the only
|
||||
// writer is the executing task itself. For a buffered run the writers
|
||||
// are external API callers, and N parallel writers exhaust 3 retries
|
||||
// quickly under contention. Bumping to 12 covers ~50-way concurrency
|
||||
// with sub-percent failure probability; the cost is bounded (each
|
||||
// retry is one Redis Lua call ~1ms).
|
||||
const maxRetries = input.maxRetries ?? 12;
|
||||
const backoffBaseMs = input.backoffBaseMs ?? 5;
|
||||
const backoffStepMs = input.backoffStepMs ?? 5;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
const entry = await buffer.getEntry(input.runId);
|
||||
if (!entry) return { kind: "not_found" };
|
||||
// Env+org check: an entry from a different env is treated as a
|
||||
// miss (not 403) so existence in other envs doesn't leak.
|
||||
if (entry.envId !== input.environmentId || entry.orgId !== input.organizationId) {
|
||||
return { kind: "not_found" };
|
||||
}
|
||||
if (entry.status !== "QUEUED" || entry.materialised) {
|
||||
return { kind: "busy" };
|
||||
}
|
||||
|
||||
const snapshot = JSON.parse(entry.payload) as Record<string, unknown>;
|
||||
const currentMetadataType =
|
||||
typeof snapshot.metadataType === "string" ? snapshot.metadataType : "application/json";
|
||||
|
||||
// Capture parent/root ids during this read so the caller can fan
|
||||
// parent/root operations out without a second buffer.getEntry. If
|
||||
// the drainer's terminal-failure path runs between our CAS-write
|
||||
// below and the route's follow-up, the entry hash would be DELd
|
||||
// and a second read would return null — silently dropping the
|
||||
// caller's `body.parentOperations` / `body.rootOperations`. The ids
|
||||
// themselves are immutable for a run, so capturing them on any
|
||||
// loop iteration is fine.
|
||||
const snapshotParentTaskRunInternalId =
|
||||
typeof snapshot.parentTaskRunId === "string" ? snapshot.parentTaskRunId : undefined;
|
||||
const snapshotParentTaskRunFriendlyId = snapshotParentTaskRunInternalId
|
||||
? RunId.toFriendlyId(snapshotParentTaskRunInternalId)
|
||||
: undefined;
|
||||
const snapshotRootTaskRunInternalId =
|
||||
typeof snapshot.rootTaskRunId === "string" ? snapshot.rootTaskRunId : undefined;
|
||||
const snapshotRootTaskRunFriendlyId = snapshotRootTaskRunInternalId
|
||||
? RunId.toFriendlyId(snapshotRootTaskRunInternalId)
|
||||
: undefined;
|
||||
|
||||
// Match PG semantics: `body.operations` and `body.metadata` are
|
||||
// mutually exclusive on a single request. The PG service
|
||||
// (`UpdateMetadataService.#updateRunMetadata`) branches on
|
||||
// `Array.isArray(body.operations)` — if operations are present it
|
||||
// applies them on top of the EXISTING metadata and ignores
|
||||
// `body.metadata` entirely; otherwise `body.metadata` is the new
|
||||
// full value. Doing both here would make a request like
|
||||
// `{ metadata: {b:2}, operations: [set c=3] }` produce
|
||||
// `{b:2,c:3}` on the buffer vs `{a:1,c:3}` on PG, which silently
|
||||
// changes semantics across the buffered/materialised boundary.
|
||||
const parseSnapshotMetadata = (): Record<string, unknown> => {
|
||||
if (typeof snapshot.metadata !== "string") return {};
|
||||
try {
|
||||
return JSON.parse(snapshot.metadata) as Record<string, unknown>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
let metadataObject: Record<string, unknown>;
|
||||
// Use `Array.isArray` (the PG service's predicate) instead of a
|
||||
// truthy length check. For `{ metadata, operations: [] }` PG sees
|
||||
// Array.isArray([])=true and no-ops on existing metadata; a
|
||||
// `.length` check would treat the empty array as falsy and fall
|
||||
// through to the `body.metadata` branch, replacing metadata —
|
||||
// exactly the cross-boundary drift the comment above warns
|
||||
// against.
|
||||
if (Array.isArray(input.body.operations)) {
|
||||
// Operations take precedence: apply on top of existing snapshot
|
||||
// metadata; ignore `body.metadata` to match PG behaviour.
|
||||
metadataObject = applyMetadataOperations(
|
||||
parseSnapshotMetadata(),
|
||||
input.body.operations
|
||||
).newMetadata;
|
||||
} else if (input.body.metadata !== undefined) {
|
||||
// No operations — full replace.
|
||||
metadataObject = input.body.metadata as Record<string, unknown>;
|
||||
} else {
|
||||
// Neither — write back existing snapshot metadata (no-op shape).
|
||||
metadataObject = parseSnapshotMetadata();
|
||||
}
|
||||
|
||||
const newMetadataStr = JSON.stringify(metadataObject);
|
||||
|
||||
// Size cap — match PG (`handleMetadataPacket` throws
|
||||
// `MetadataTooLargeError` (413) when the JSON-encoded packet
|
||||
// exceeds the configured cap). Reject in-loop, before CAS, so a
|
||||
// single oversize write doesn't churn the retry budget.
|
||||
const observedSize = Buffer.byteLength(newMetadataStr, "utf8");
|
||||
if (observedSize > input.maximumSize) {
|
||||
return {
|
||||
kind: "metadata_too_large",
|
||||
maximumSize: input.maximumSize,
|
||||
observedSize,
|
||||
};
|
||||
}
|
||||
|
||||
const cas = await buffer.casSetMetadata({
|
||||
runId: input.runId,
|
||||
expectedVersion: entry.metadataVersion,
|
||||
newMetadata: newMetadataStr,
|
||||
newMetadataType: currentMetadataType,
|
||||
});
|
||||
|
||||
if (cas.kind === "applied") {
|
||||
return {
|
||||
kind: "applied",
|
||||
newMetadata: metadataObject,
|
||||
parentTaskRunFriendlyId: snapshotParentTaskRunFriendlyId,
|
||||
rootTaskRunFriendlyId: snapshotRootTaskRunFriendlyId,
|
||||
};
|
||||
}
|
||||
if (cas.kind === "not_found") return { kind: "not_found" };
|
||||
if (cas.kind === "busy") return { kind: "busy" };
|
||||
// version_conflict — another caller wrote between our read + CAS.
|
||||
// Small jittered backoff so a thundering herd of N retriers doesn't
|
||||
// all re-read + re-CAS at exactly the same moment.
|
||||
logger.debug("applyMetadataMutationToBufferedRun: version_conflict, retrying", {
|
||||
runId: input.runId,
|
||||
attempt,
|
||||
observedVersion: entry.metadataVersion,
|
||||
currentVersion: cas.currentVersion,
|
||||
});
|
||||
const backoffMs = Math.floor(Math.random() * (backoffBaseMs + attempt * backoffStepMs));
|
||||
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
||||
}
|
||||
|
||||
logger.warn("applyMetadataMutationToBufferedRun: retries exhausted", {
|
||||
runId: input.runId,
|
||||
maxRetries,
|
||||
});
|
||||
return { kind: "version_exhausted" };
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { TriggerTaskRequestBody } from "@trigger.dev/core/v3";
|
||||
import type { TriggerTaskServiceOptions } from "~/v3/services/triggerTask.server";
|
||||
|
||||
// Canonical payload shape written to the mollifier buffer when the gate
|
||||
// decides to mollify a trigger. At this stage the call site ALSO calls
|
||||
// engine.trigger directly (dual-write), so this is currently an
|
||||
// audit/preview record. A later change makes the buffer the primary write
|
||||
// path: the drainer's handler reads this payload and replays it through
|
||||
// engine.trigger to create the run in Postgres, and read-fallback
|
||||
// endpoints synthesise a Run view from it while it is still QUEUED.
|
||||
//
|
||||
// CONTRACT: this shape must contain everything the drainer-replay needs to
|
||||
// reconstruct an equivalent engine.trigger call. Today it is emitted to
|
||||
// logs; later it is serialised into Redis and rebuilt on the drain side.
|
||||
// Keep it serialisable — no functions, no class instances.
|
||||
export type BufferedTriggerPayload = {
|
||||
runFriendlyId: string;
|
||||
|
||||
// Routing identifiers — let the drainer re-fetch full AuthenticatedEnvironment
|
||||
// at replay time rather than embedding it in the payload.
|
||||
envId: string;
|
||||
envType: string;
|
||||
envSlug: string;
|
||||
orgId: string;
|
||||
orgSlug: string;
|
||||
projectId: string;
|
||||
projectRef: string;
|
||||
|
||||
// Task identifier — looked up against the locked BackgroundWorkerTask
|
||||
// at replay time to recover task-defaults.
|
||||
taskId: string;
|
||||
|
||||
// Customer-supplied trigger body (payload, options, context).
|
||||
body: TriggerTaskRequestBody;
|
||||
|
||||
// Resolved values from upstream concerns. The drainer should NOT re-resolve
|
||||
// these — that would create a second idempotency-key check, etc.
|
||||
idempotencyKey: string | null;
|
||||
idempotencyKeyExpiresAt: string | null;
|
||||
tags: string[];
|
||||
|
||||
// Parent/root linkage for nested triggers.
|
||||
parentRunFriendlyId: string | null;
|
||||
|
||||
// Trace context — propagates the original triggering span across the
|
||||
// buffer→drain boundary so the run's lifecycle stays under one trace.
|
||||
traceContext: Record<string, unknown>;
|
||||
|
||||
// Annotations + service options that influence routing/replay.
|
||||
triggerSource: string;
|
||||
triggerAction: string;
|
||||
serviceOptions: TriggerTaskServiceOptions;
|
||||
|
||||
// Wall-clock instants relevant to the run.
|
||||
createdAt: string;
|
||||
};
|
||||
|
||||
// Assemble the canonical payload from the inputs available at the point
|
||||
// `evaluateGate` returns "mollify" in `RunEngineTriggerTaskService.call`.
|
||||
// All fields must be derivable from data already in scope at that call site;
|
||||
// nothing should require an extra DB lookup.
|
||||
export function buildBufferedTriggerPayload(input: {
|
||||
runFriendlyId: string;
|
||||
taskId: string;
|
||||
envId: string;
|
||||
envType: string;
|
||||
envSlug: string;
|
||||
orgId: string;
|
||||
orgSlug: string;
|
||||
projectId: string;
|
||||
projectRef: string;
|
||||
body: TriggerTaskRequestBody;
|
||||
idempotencyKey: string | null;
|
||||
idempotencyKeyExpiresAt: Date | null;
|
||||
tags: string[];
|
||||
parentRunFriendlyId: string | null;
|
||||
traceContext: Record<string, unknown>;
|
||||
triggerSource: string;
|
||||
triggerAction: string;
|
||||
serviceOptions: TriggerTaskServiceOptions;
|
||||
createdAt: Date;
|
||||
}): BufferedTriggerPayload {
|
||||
return {
|
||||
runFriendlyId: input.runFriendlyId,
|
||||
envId: input.envId,
|
||||
envType: input.envType,
|
||||
envSlug: input.envSlug,
|
||||
orgId: input.orgId,
|
||||
orgSlug: input.orgSlug,
|
||||
projectId: input.projectId,
|
||||
projectRef: input.projectRef,
|
||||
taskId: input.taskId,
|
||||
body: input.body,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
idempotencyKeyExpiresAt:
|
||||
input.idempotencyKey && input.idempotencyKeyExpiresAt
|
||||
? input.idempotencyKeyExpiresAt.toISOString()
|
||||
: null,
|
||||
tags: input.tags,
|
||||
parentRunFriendlyId: input.parentRunFriendlyId,
|
||||
traceContext: input.traceContext,
|
||||
triggerSource: input.triggerSource,
|
||||
triggerAction: input.triggerAction,
|
||||
serviceOptions: input.serviceOptions,
|
||||
createdAt: input.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type {
|
||||
IdempotencyClaimResult,
|
||||
IdempotencyLookupInput,
|
||||
MollifierBuffer,
|
||||
} from "@trigger.dev/redis-worker";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
|
||||
// Tunables. The TTL on the claim key is bounded by typical trigger-pipeline
|
||||
// dwell; long enough that a slow PG insert doesn't expire mid-flight,
|
||||
// short enough that a crashed claimant unblocks waiters quickly.
|
||||
export const DEFAULT_CLAIM_TTL_SECONDS = 30;
|
||||
// safetyNetMs caps how long a waiter blocks before returning timed_out.
|
||||
// Matches the mutateWithFallback safety net so SDK retry policies don't
|
||||
// have to special-case this path.
|
||||
export const DEFAULT_CLAIM_WAIT_MS = 5_000;
|
||||
export const DEFAULT_CLAIM_POLL_MS = 25;
|
||||
|
||||
export type ClaimOrAwaitOutcome =
|
||||
// We own the claim. `token` MUST be passed to publishClaim/releaseClaim
|
||||
// so the buffer can compare-and-act against our ownership marker — a
|
||||
// late release from a previous claimant whose TTL expired cannot
|
||||
// erase our slot.
|
||||
| { kind: "claimed"; token: string }
|
||||
| { kind: "resolved"; runId: string } // someone else's runId; caller returns isCached:true
|
||||
| { kind: "timed_out" };
|
||||
|
||||
export type ClaimOrAwaitInput = IdempotencyLookupInput & {
|
||||
ttlSeconds?: number;
|
||||
safetyNetMs?: number;
|
||||
pollStepMs?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
// Test injection.
|
||||
buffer?: MollifierBuffer | null;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
// Test override for the ownership-token generator. Defaults to
|
||||
// `crypto.randomUUID()`. Tests pass a deterministic value so they
|
||||
// can assert publish/release pass-through.
|
||||
generateToken?: () => string;
|
||||
};
|
||||
|
||||
// Pre-gate Redis claim. All same-key triggers serialise through here
|
||||
// before the trigger pipeline runs. Returning `resolved` short-circuits
|
||||
// the trigger entirely — the caller responds with the cached runId.
|
||||
// Returning `claimed` means we own the claim and MUST publish the
|
||||
// winning runId on success (`publishClaim`) or release the claim on
|
||||
// failure (`releaseClaim`).
|
||||
//
|
||||
// Failure modes:
|
||||
// - Redis down at claim time: returns `claimed` (fail open, no
|
||||
// coordination). Customer is no worse than today's race; the
|
||||
// PG unique constraint is the eventual arbiter.
|
||||
// - Claimant crashes mid-pipeline: claim TTL expires, waiters
|
||||
// eventually time out, SDK retries.
|
||||
// - PG/buffer publish failure: waiters time out and SDK retries; next
|
||||
// attempt sees the eventual PG/buffer state via existing
|
||||
// IdempotencyKeyConcern PG-first lookup.
|
||||
export async function claimOrAwait(input: ClaimOrAwaitInput): Promise<ClaimOrAwaitOutcome> {
|
||||
const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
|
||||
if (!buffer) {
|
||||
// Mollifier disabled / buffer construction failed. Fall open —
|
||||
// caller proceeds with the trigger pipeline (PG unique constraint
|
||||
// backstop). The token is never read in this case (publish/release
|
||||
// are buffer-null no-ops downstream), so we skip the default
|
||||
// `randomUUID()` to keep the mollifier-OFF hot path allocation-free
|
||||
// for idempotency-keyed triggers — `triggerTask` is the
|
||||
// highest-throughput code path in the system. A test-injected
|
||||
// generator is still honoured for deterministic assertions.
|
||||
return { kind: "claimed", token: input.generateToken ? input.generateToken() : "" };
|
||||
}
|
||||
const generateToken = input.generateToken ?? randomUUID;
|
||||
// Generate the ownership token up front so the retry loop reuses it
|
||||
// — we're the same logical claimant across attempts; only the slot
|
||||
// owner changes between releases.
|
||||
const token = generateToken();
|
||||
const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS;
|
||||
const safetyNetMs = input.safetyNetMs ?? DEFAULT_CLAIM_WAIT_MS;
|
||||
const pollStepMs = input.pollStepMs ?? DEFAULT_CLAIM_POLL_MS;
|
||||
const now = input.now ?? Date.now;
|
||||
const sleep = input.sleep ?? defaultSleep;
|
||||
|
||||
const lookupInput: IdempotencyLookupInput = {
|
||||
envId: input.envId,
|
||||
taskIdentifier: input.taskIdentifier,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
};
|
||||
|
||||
// Initial claim attempt. Most production-path calls resolve here on
|
||||
// the first call (either we win, or the key is already resolved from
|
||||
// a prior burst).
|
||||
let result: IdempotencyClaimResult;
|
||||
try {
|
||||
result = await buffer.claimIdempotency({ ...lookupInput, token, ttlSeconds });
|
||||
} catch (err) {
|
||||
logger.warn("idempotency claim failed (fail-open)", {
|
||||
envId: input.envId,
|
||||
taskIdentifier: input.taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return { kind: "claimed", token };
|
||||
}
|
||||
|
||||
if (result.kind === "claimed") return { kind: "claimed", token };
|
||||
if (result.kind === "resolved") return result;
|
||||
|
||||
// result.kind === "pending" — wait/poll loop. May see the value flip
|
||||
// to "resolved" (winner published), the key vanish (winner released
|
||||
// on error → retry claim), or stay "pending" until the safety net.
|
||||
const deadline = now() + safetyNetMs;
|
||||
while (now() < deadline) {
|
||||
if (input.abortSignal?.aborted) return { kind: "timed_out" };
|
||||
await sleep(pollStepMs);
|
||||
|
||||
let current: IdempotencyClaimResult | null;
|
||||
try {
|
||||
current = await buffer.readClaim(lookupInput);
|
||||
} catch (err) {
|
||||
// Transient read failure — keep polling until deadline.
|
||||
logger.warn("idempotency claim read failed mid-poll", {
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current === null) {
|
||||
// Claimant released on error. Re-attempt the claim — one of the
|
||||
// waiters will win, the rest see "pending" again. Reuse our token:
|
||||
// we're still the same logical claimant, just contending for a
|
||||
// freshly empty slot.
|
||||
try {
|
||||
const retry = await buffer.claimIdempotency({ ...lookupInput, token, ttlSeconds });
|
||||
if (retry.kind === "claimed") return { kind: "claimed", token };
|
||||
if (retry.kind === "resolved") return retry;
|
||||
// "pending" again → keep polling.
|
||||
} catch (err) {
|
||||
logger.warn("idempotency claim retry failed", {
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return { kind: "claimed", token };
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (current.kind === "resolved") return current;
|
||||
// current.kind === "pending" → keep polling.
|
||||
}
|
||||
return { kind: "timed_out" };
|
||||
}
|
||||
|
||||
// Publish the winning runId so waiters resolve. Best-effort: failure
|
||||
// here means waiters will time out and the SDK will retry, which will
|
||||
// then find the row via the existing IdempotencyKeyConcern PG-first
|
||||
// check.
|
||||
export async function publishClaim(input: {
|
||||
envId: string;
|
||||
taskIdentifier: string;
|
||||
idempotencyKey: string;
|
||||
// Ownership token from the `claimed` outcome. Buffer compare-and-sets
|
||||
// on this so a publish from a stale claimant (TTL expired, another
|
||||
// claimant moved in) is a no-op rather than overwriting their claim.
|
||||
token: string;
|
||||
runId: string;
|
||||
ttlSeconds?: number;
|
||||
buffer?: MollifierBuffer | null;
|
||||
}): Promise<void> {
|
||||
const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
|
||||
if (!buffer) return;
|
||||
const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS;
|
||||
try {
|
||||
await buffer.publishClaim({
|
||||
envId: input.envId,
|
||||
taskIdentifier: input.taskIdentifier,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
token: input.token,
|
||||
runId: input.runId,
|
||||
ttlSeconds,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn("idempotency claim publish failed", {
|
||||
envId: input.envId,
|
||||
taskIdentifier: input.taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Release on pipeline failure. Best-effort. If the DEL fails, the claim
|
||||
// TTL is the safety net — waiters time out, SDK retries.
|
||||
export async function releaseClaim(input: {
|
||||
envId: string;
|
||||
taskIdentifier: string;
|
||||
idempotencyKey: string;
|
||||
// Ownership token from the `claimed` outcome. Buffer compare-and-
|
||||
// deletes on this so a release from a stale claimant whose TTL
|
||||
// expired can't wipe a new owner's claim.
|
||||
token: string;
|
||||
buffer?: MollifierBuffer | null;
|
||||
}): Promise<void> {
|
||||
const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
|
||||
if (!buffer) return;
|
||||
try {
|
||||
await buffer.releaseClaim({
|
||||
envId: input.envId,
|
||||
taskIdentifier: input.taskIdentifier,
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
token: input.token,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn("idempotency claim release failed", {
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function defaultSleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
// DI seam type for consumers (e.g. triggerTask.server.ts) that need a
|
||||
// nullable buffer accessor at construction time.
|
||||
export type MollifierGetBuffer = () => MollifierBuffer | null;
|
||||
|
||||
function initializeMollifierBuffer(): MollifierBuffer {
|
||||
logger.debug("Initializing mollifier buffer", {
|
||||
host: env.TRIGGER_MOLLIFIER_REDIS_HOST,
|
||||
});
|
||||
|
||||
return new MollifierBuffer({
|
||||
redisOptions: {
|
||||
keyPrefix: "",
|
||||
host: env.TRIGGER_MOLLIFIER_REDIS_HOST,
|
||||
port: env.TRIGGER_MOLLIFIER_REDIS_PORT,
|
||||
username: env.TRIGGER_MOLLIFIER_REDIS_USERNAME,
|
||||
password: env.TRIGGER_MOLLIFIER_REDIS_PASSWORD,
|
||||
enableAutoPipelining: true,
|
||||
...(env.TRIGGER_MOLLIFIER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
ackGraceTtlSeconds: env.TRIGGER_MOLLIFIER_ACK_GRACE_TTL_SECONDS,
|
||||
maxRetriesPerRequest: env.TRIGGER_MOLLIFIER_REDIS_MAX_RETRIES_PER_REQUEST,
|
||||
reconnectStepMs: env.TRIGGER_MOLLIFIER_REDIS_RECONNECT_STEP_MS,
|
||||
reconnectMaxMs: env.TRIGGER_MOLLIFIER_REDIS_RECONNECT_MAX_MS,
|
||||
});
|
||||
}
|
||||
|
||||
export function getMollifierBuffer(): MollifierBuffer | null {
|
||||
if (env.TRIGGER_MOLLIFIER_ENABLED !== "1") return null;
|
||||
return singleton("mollifierBuffer", initializeMollifierBuffer);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { MollifierDrainer } from "@trigger.dev/redis-worker";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { engine as runEngine } from "~/v3/runEngine.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
import {
|
||||
createDrainerHandler,
|
||||
createDrainerTerminalFailureHandler,
|
||||
isRetryablePgError,
|
||||
} from "./mollifierDrainerHandler.server";
|
||||
import type { MollifierSnapshot } from "./mollifierSnapshot.server";
|
||||
|
||||
// Distinct error class for the deterministic "fail loud at boot" throws
|
||||
// below. The bootstrap in `mollifierDrainerWorker.server.ts` catches
|
||||
// transient/init errors and logs them so an unrelated Redis blip doesn't
|
||||
// crash the webapp, but it RETHROWS this class — a misconfigured
|
||||
// shutdown timeout or missing buffer is a deploy-time mistake that
|
||||
// should fail health checks and roll back, not silently disable a
|
||||
// half-rolled-out feature.
|
||||
//
|
||||
// The `name` getter is set explicitly so cross-realm `instanceof` checks
|
||||
// (e.g. when Remix dev hot-reloads the module and the consumer keeps a
|
||||
// reference to the old class) can fall back to `error.name === ...` and
|
||||
// still recognise the marker.
|
||||
export class MollifierConfigurationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "MollifierConfigurationError";
|
||||
}
|
||||
}
|
||||
|
||||
function initializeMollifierDrainer(): MollifierDrainer<MollifierSnapshot> {
|
||||
const buffer = getMollifierBuffer();
|
||||
if (!buffer) {
|
||||
// Unreachable in normal config: getMollifierDrainer() gates on the
|
||||
// same env flag as getMollifierBuffer(). If we hit this, fail loud
|
||||
// — the operator has set TRIGGER_MOLLIFIER_ENABLED=1 on a worker pod but
|
||||
// the buffer can't initialise (e.g. TRIGGER_MOLLIFIER_REDIS_HOST resolves
|
||||
// to nothing). Crashing surfaces the misconfig immediately rather
|
||||
// than silently leaving entries un-drained.
|
||||
throw new MollifierConfigurationError(
|
||||
"MollifierDrainer initialised without a buffer — env vars inconsistent"
|
||||
);
|
||||
}
|
||||
|
||||
// Validate BEFORE start() so a misconfigured shutdown timeout fails
|
||||
// loud at module-load time and the singleton is never cached. If start()
|
||||
// ran first and the throw propagated out, the loop would already be
|
||||
// polling with no SIGTERM handler registered by the caller — exactly
|
||||
// the failure mode the validation is supposed to prevent.
|
||||
//
|
||||
// The SIGTERM handler in mollifierDrainerWorker.server.ts is sync fire-and-forget:
|
||||
// `drainer.stop({ timeoutMs })` returns a promise that keeps the event
|
||||
// loop alive, but in cluster mode the primary runs its own
|
||||
// GRACEFUL_SHUTDOWN_TIMEOUT and will call `process.exit(0)`
|
||||
// independently. If the drainer's deadline exceeds the primary's, the
|
||||
// drainer is cut off mid-wait — "log a warning on timeout" turns into
|
||||
// "hard exit with no log". 1s margin gives the primary room to finish
|
||||
// its own teardown after the drainer settles.
|
||||
const shutdownMarginMs = env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_MARGIN_MS;
|
||||
if (
|
||||
env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS >=
|
||||
env.GRACEFUL_SHUTDOWN_TIMEOUT - shutdownMarginMs
|
||||
) {
|
||||
throw new MollifierConfigurationError(
|
||||
`TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS (${env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS}) must be at least ${shutdownMarginMs}ms below GRACEFUL_SHUTDOWN_TIMEOUT (${env.GRACEFUL_SHUTDOWN_TIMEOUT}); otherwise the primary's hard exit shadows the drainer's deadline.`
|
||||
);
|
||||
}
|
||||
|
||||
logger.debug("Initializing mollifier drainer", {
|
||||
concurrency: env.TRIGGER_MOLLIFIER_DRAIN_CONCURRENCY,
|
||||
maxAttempts: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ATTEMPTS,
|
||||
drainBatchSize: env.TRIGGER_MOLLIFIER_DRAIN_BATCH_SIZE,
|
||||
});
|
||||
|
||||
const drainer = new MollifierDrainer<MollifierSnapshot>({
|
||||
buffer,
|
||||
handler: createDrainerHandler({ engine: runEngine, prisma }),
|
||||
onTerminalFailure: createDrainerTerminalFailureHandler({ engine: runEngine, prisma }),
|
||||
concurrency: env.TRIGGER_MOLLIFIER_DRAIN_CONCURRENCY,
|
||||
maxAttempts: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ATTEMPTS,
|
||||
maxOrgsPerTick: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ORGS_PER_TICK,
|
||||
drainBatchSize: env.TRIGGER_MOLLIFIER_DRAIN_BATCH_SIZE,
|
||||
pollIntervalMs: env.TRIGGER_MOLLIFIER_DRAIN_POLL_INTERVAL_MS,
|
||||
maxBackoffMs: env.TRIGGER_MOLLIFIER_DRAIN_MAX_BACKOFF_MS,
|
||||
backoffFloorMs: env.TRIGGER_MOLLIFIER_DRAIN_BACKOFF_FLOOR_MS,
|
||||
isRetryable: isRetryablePgError,
|
||||
});
|
||||
|
||||
return drainer;
|
||||
}
|
||||
|
||||
// Returns a configured-but-stopped drainer. Callers MUST register their
|
||||
// SIGTERM / SIGINT shutdown handlers before invoking `drainer.start()` —
|
||||
// see `apps/webapp/app/v3/mollifierDrainerWorker.server.ts`. Starting
|
||||
// inside the singleton factory would put the polling loop ahead of
|
||||
// handler registration, leaving a narrow window where a SIGTERM landing
|
||||
// between `start()` and `process.once("SIGTERM", ...)` would skip the
|
||||
// graceful stop. The split is intentional.
|
||||
export function getMollifierDrainer(): MollifierDrainer<MollifierSnapshot> | null {
|
||||
if (env.TRIGGER_MOLLIFIER_ENABLED !== "1") return null;
|
||||
return singleton("mollifierDrainer", initializeMollifierDrainer);
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
import { context, trace, TraceFlags } from "@opentelemetry/api";
|
||||
import type { RunEngine } from "@internal/run-engine";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type {
|
||||
MollifierDrainerHandler,
|
||||
MollifierDrainerTerminalFailureHandler,
|
||||
} from "@trigger.dev/redis-worker";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { recordRunDebugLog } from "~/v3/eventRepository/index.server";
|
||||
import { PerformTaskRunAlertsService } from "~/v3/services/alerts/performTaskRunAlerts.server";
|
||||
import { startSpan } from "~/v3/tracing.server";
|
||||
import type { MollifierSnapshot } from "./mollifierSnapshot.server";
|
||||
|
||||
const tracer = trace.getTracer("mollifier-drainer");
|
||||
|
||||
export function isRetryablePgError(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) return false;
|
||||
const msg = err.message ?? "";
|
||||
// Prisma surfaces P1001 ("Can't reach database server") via two
|
||||
// different error classes — `PrismaClientKnownRequestError` exposes
|
||||
// it as `err.code`, `PrismaClientInitializationError` exposes it as
|
||||
// `err.errorCode`. Check both so reconnection-time errors retry
|
||||
// regardless of which class fires.
|
||||
const code = (err as { code?: string }).code;
|
||||
const errorCode = (err as { errorCode?: string }).errorCode;
|
||||
if (code === "P2024") return true;
|
||||
if (code === "P1001" || errorCode === "P1001") return true;
|
||||
if (msg.includes("Can't reach database server")) return true;
|
||||
if (msg.includes("Connection lost")) return true;
|
||||
if (msg.includes("ECONNRESET")) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function createDrainerHandler(deps: {
|
||||
engine: RunEngine;
|
||||
prisma: PrismaClientOrTransaction;
|
||||
}): MollifierDrainerHandler<MollifierSnapshot> {
|
||||
return async (input) => {
|
||||
const dwellMs = Date.now() - input.createdAt.getTime();
|
||||
|
||||
// Re-attach to the trace started by the caller's mollifier.queued span
|
||||
// (its traceId + spanId were captured into the snapshot at buffer time).
|
||||
// Without this the drainer would emit mollifier.drained in a brand-new
|
||||
// trace and the engine.trigger instrumentation would inherit an empty
|
||||
// active context — leaving the run-detail page with only the root span.
|
||||
const snapshotTraceId =
|
||||
typeof input.payload.traceId === "string" ? input.payload.traceId : undefined;
|
||||
const snapshotSpanId =
|
||||
typeof input.payload.spanId === "string" ? input.payload.spanId : undefined;
|
||||
|
||||
const parentContext =
|
||||
snapshotTraceId && snapshotSpanId
|
||||
? trace.setSpanContext(context.active(), {
|
||||
traceId: snapshotTraceId,
|
||||
spanId: snapshotSpanId,
|
||||
traceFlags: TraceFlags.SAMPLED,
|
||||
isRemote: true,
|
||||
})
|
||||
: context.active();
|
||||
|
||||
// Cancel-wins-over-trigger. If a cancel API call landed on this
|
||||
// entry while it was QUEUED, the snapshot carries `cancelledAt` +
|
||||
// `cancelReason`. Skip the normal materialise path and write a
|
||||
// CANCELED PG row directly. The `runCancelled` bus emit is
|
||||
// suppressed here because a buffered-only run never had a primary
|
||||
// trace event written for it — the runCancelled handler's
|
||||
// `cancelRunEvent` lookup would fail and log noise per cancel.
|
||||
const cancelledAtStr =
|
||||
typeof input.payload.cancelledAt === "string" ? input.payload.cancelledAt : undefined;
|
||||
if (cancelledAtStr) {
|
||||
const cancelReason =
|
||||
typeof input.payload.cancelReason === "string"
|
||||
? input.payload.cancelReason
|
||||
: "Canceled by user";
|
||||
await context.with(parentContext, async () => {
|
||||
await startSpan(tracer, "mollifier.drained.cancelled", async (span) => {
|
||||
span.setAttribute("mollifier.drained", true);
|
||||
span.setAttribute("mollifier.dwell_ms", dwellMs);
|
||||
span.setAttribute("mollifier.attempts", input.attempts);
|
||||
span.setAttribute("mollifier.run_friendly_id", input.runId);
|
||||
span.setAttribute("mollifier.cancel_bifurcation", true);
|
||||
span.setAttribute("taskRunId", input.runId);
|
||||
try {
|
||||
await deps.engine.createCancelledRun(
|
||||
{
|
||||
snapshot: input.payload as any,
|
||||
cancelledAt: new Date(cancelledAtStr),
|
||||
cancelReason,
|
||||
emitRunCancelledEvent: false,
|
||||
},
|
||||
deps.prisma
|
||||
);
|
||||
} catch (err) {
|
||||
// createCancelledRun throws a conflict when the normal trigger
|
||||
// replay path won the race and already materialised a live
|
||||
// (non-CANCELED) row for this friendlyId. Its contract leaves
|
||||
// the resolution to us: honour the cancel by actually
|
||||
// cancelling the now-live run. Letting the conflict propagate
|
||||
// would instead reach the drainer's terminal-failure path
|
||||
// (isRetryablePgError() is false for it), buffer.fail() the
|
||||
// entry, and silently lose the cancellation while the run
|
||||
// keeps executing.
|
||||
const isConflict =
|
||||
err instanceof Error && err.message.startsWith("createCancelledRun conflict");
|
||||
if (!isConflict) {
|
||||
// Mirror the SYSTEM_FAILURE fallback the non-cancelled
|
||||
// trigger path uses below. Without this branch, a
|
||||
// non-retryable createCancelledRun failure rethrows, the
|
||||
// drainer's onTerminalFailure handler skips because it
|
||||
// gates on `cause === "max-attempts-exhausted"` (and the
|
||||
// outer drainer classifies non-retryable failures with
|
||||
// `cause: "non-retryable"`), and buffer.fail() deletes
|
||||
// the entry — leaving NO PG row. The cancellation
|
||||
// disappears silently from the customer's dashboard.
|
||||
// Writing a SYSTEM_FAILURE row gives the run a terminal,
|
||||
// visible state.
|
||||
if (isRetryablePgError(err)) {
|
||||
throw err;
|
||||
}
|
||||
span.setAttribute(
|
||||
"mollifier.cancel_terminal_failure_reason",
|
||||
err instanceof Error ? err.message : String(err)
|
||||
);
|
||||
try {
|
||||
const wrote = await writeMollifierTerminalFailureRow(deps, {
|
||||
friendlyId: input.runId,
|
||||
snapshot: input.payload as Record<string, unknown>,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
if (wrote) return;
|
||||
} catch (writeErr) {
|
||||
if (isRetryablePgError(writeErr)) {
|
||||
span.setAttribute("mollifier.cancel_terminal_write_retryable", true);
|
||||
throw writeErr;
|
||||
}
|
||||
span.setAttribute(
|
||||
"mollifier.cancel_terminal_write_error",
|
||||
writeErr instanceof Error ? writeErr.message : String(writeErr)
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
span.setAttribute("mollifier.cancel_conflict", true);
|
||||
const friendlyId =
|
||||
typeof input.payload.friendlyId === "string" ? input.payload.friendlyId : input.runId;
|
||||
await deps.engine.cancelRun({
|
||||
runId: RunId.fromFriendlyId(friendlyId),
|
||||
completedAt: new Date(cancelledAtStr),
|
||||
reason: cancelReason,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await context.with(parentContext, async () => {
|
||||
await startSpan(tracer, "mollifier.drained", async (span) => {
|
||||
span.setAttribute("mollifier.drained", true);
|
||||
span.setAttribute("mollifier.dwell_ms", dwellMs);
|
||||
span.setAttribute("mollifier.attempts", input.attempts);
|
||||
span.setAttribute("mollifier.run_friendly_id", input.runId);
|
||||
span.setAttribute("taskRunId", input.runId);
|
||||
|
||||
let triggerSucceeded = false;
|
||||
try {
|
||||
await deps.engine.trigger(input.payload as any, deps.prisma);
|
||||
triggerSucceeded = true;
|
||||
} catch (err) {
|
||||
// The retryable-PG class re-throws so the drainer's outer
|
||||
// worker loop can `buffer.requeue` (handled in
|
||||
// `MollifierDrainer.drainOne`). For non-retryable failures we
|
||||
// write a terminal SYSTEM_FAILURE row to PG via the engine's
|
||||
// existing `createFailedTaskRun` (used by batch-trigger for
|
||||
// the same purpose) so the customer sees the run in their
|
||||
// dashboard / SDK instead of silently losing it when the
|
||||
// buffer entry TTLs out. If THAT insert also fails (PG truly
|
||||
// unreachable), rethrow so the drainer's outer catch falls
|
||||
// through to its existing `buffer.fail` terminal-marker path.
|
||||
if (isRetryablePgError(err)) {
|
||||
throw err;
|
||||
}
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
span.setAttribute("mollifier.terminal_failure_reason", reason);
|
||||
try {
|
||||
const wrote = await writeMollifierTerminalFailureRow(deps, {
|
||||
friendlyId: input.runId,
|
||||
snapshot: input.payload as Record<string, unknown>,
|
||||
reason,
|
||||
});
|
||||
if (!wrote) {
|
||||
// Snapshot too malformed to even construct a TaskRun row.
|
||||
// Drainer's outer catch will buffer.fail this entry.
|
||||
throw err;
|
||||
}
|
||||
} catch (writeErr) {
|
||||
// The terminal SYSTEM_FAILURE write itself failed. If it
|
||||
// failed because PG is transiently unreachable, rethrow the
|
||||
// *write* error so the drainer requeues — buffer.fail()ing on
|
||||
// the original non-retryable error would lose the run with no
|
||||
// PG row ever landing. Once PG recovers the requeued entry
|
||||
// writes its failure row and the customer sees it.
|
||||
if (isRetryablePgError(writeErr)) {
|
||||
span.setAttribute("mollifier.terminal_write_retryable", true);
|
||||
throw writeErr;
|
||||
}
|
||||
// PG reachable but the write was rejected for another reason
|
||||
// (genuinely bad snapshot). Rethrow the original trigger error
|
||||
// so the drainer falls back to buffer.fail.
|
||||
span.setAttribute(
|
||||
"mollifier.terminal_write_error",
|
||||
writeErr instanceof Error ? writeErr.message : String(writeErr)
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Admin-only audit trail emitted once engine.trigger has
|
||||
// landed a PG row. `recordRunDebugLog` flips this to the
|
||||
// admin-gated debug kind (TaskEventKind.LOG in the PG store /
|
||||
// DEBUG_EVENT in the ClickHouse store) which the trace view +
|
||||
// logs download already strip for non-admins
|
||||
// (`eventRepository.server.ts:108`,
|
||||
// `resources.runs.$runParam.logs.download.ts:118`).
|
||||
//
|
||||
// Placement: emit as a zero-duration marker AT materialisation
|
||||
// time, not as a back-dated bar spanning the buffered window.
|
||||
// `engine.trigger` rewrites the run's root span at
|
||||
// materialisation (it adopts the synth root via traceId/spanId
|
||||
// carryover but updates start_time to "now"), so the trace
|
||||
// renderer treats materialisation time as t=0. A back-dated
|
||||
// event with startTime = bufferedAt would land before that t=0
|
||||
// and get clipped from the tree. Same pattern as the
|
||||
// `[engine] QUEUED` markers. The window itself is preserved
|
||||
// in metadata so admins can read it off the span detail pane.
|
||||
//
|
||||
// Best-effort: `recordRunDebugLog` has its own try/catch and
|
||||
// returns a result, so it never throws into the materialisation
|
||||
// path. Failures are logged but not surfaced because the
|
||||
// customer-visible run has already landed.
|
||||
if (triggerSucceeded) {
|
||||
const debugResult = await recordRunDebugLog(
|
||||
RunId.fromFriendlyId(input.runId),
|
||||
`Mollifier buffered ${dwellMs}ms before materialising`,
|
||||
{
|
||||
attributes: {
|
||||
runId: input.runId,
|
||||
metadata: {
|
||||
"mollifier.bufferedAt": input.createdAt.toISOString(),
|
||||
"mollifier.materialisedAt": new Date().toISOString(),
|
||||
"mollifier.dwellMs": dwellMs,
|
||||
"mollifier.attempts": input.attempts,
|
||||
},
|
||||
},
|
||||
parentId: snapshotSpanId,
|
||||
}
|
||||
);
|
||||
if (!debugResult.success && debugResult.code !== "RUN_NOT_FOUND") {
|
||||
logger.warn("mollifier drainer: failed to record admin debug log", {
|
||||
runId: input.runId,
|
||||
code: debugResult.code,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// Shared SYSTEM_FAILURE construction used by both terminal paths:
|
||||
// - non-retryable failure inside the handler (above)
|
||||
// - retryable failure after maxAttempts inside the drainer's
|
||||
// `processEntry` (via `createDrainerTerminalFailureHandler`)
|
||||
//
|
||||
// Suppresses `runFailed` and enqueues the alert manually — the engine's
|
||||
// `runFailed` handler calls `completeFailedRunEvent`, which looks up
|
||||
// the run's primary span. Buffered-only runs never had a primary trace
|
||||
// event written (the mollifier gate intercepts BEFORE
|
||||
// `repository.traceEvent` runs), so the lookup always fails and the
|
||||
// handler logs a systematic `[runFailed] Failed to complete failed
|
||||
// run event` error per terminal failure. `TriggerFailedTaskService`
|
||||
// handles the identical situation the same way (see triggerFailedTask
|
||||
// .server.ts:212 and 324) — pass `emitRunFailedEvent: false` to the
|
||||
// engine and call `PerformTaskRunAlertsService.enqueue(...)` directly
|
||||
// so customers' ERROR channels still fire. Alert enqueue is
|
||||
// best-effort; an alert-side failure is logged but does not bubble up
|
||||
// (the SYSTEM_FAILURE row landing is the load-bearing customer-visible
|
||||
// outcome).
|
||||
//
|
||||
// Returns the new `TaskRun` on success or `null` when the snapshot was
|
||||
// so malformed it couldn't even produce an environment — caller decides
|
||||
// whether to escalate that to `buffer.fail` directly. Throws on any
|
||||
// other failure so the drainer's retryable/non-retryable disposition
|
||||
// logic can own the decision.
|
||||
async function writeMollifierTerminalFailureRow(
|
||||
deps: { engine: RunEngine; prisma: PrismaClientOrTransaction },
|
||||
args: { friendlyId: string; snapshot: Record<string, unknown>; reason: string }
|
||||
) {
|
||||
const { snapshot } = args;
|
||||
const env = snapshot.environment as
|
||||
| {
|
||||
id: string;
|
||||
type: any;
|
||||
project: { id: string };
|
||||
organization: { id: string };
|
||||
}
|
||||
| undefined;
|
||||
if (!env) return null;
|
||||
// Extract batch association from the snapshot if present. Without this
|
||||
// a SYSTEM_FAILURE row for a buffered batch child won't be linked to
|
||||
// its batch, and the batch parent's completion tracking can hang
|
||||
// indefinitely waiting on a child that landed but isn't visible to
|
||||
// the batch.
|
||||
const rawBatch = snapshot.batch;
|
||||
const batch =
|
||||
rawBatch &&
|
||||
typeof rawBatch === "object" &&
|
||||
"id" in rawBatch &&
|
||||
typeof (rawBatch as { id: unknown }).id === "string" &&
|
||||
"index" in rawBatch &&
|
||||
typeof (rawBatch as { index: unknown }).index === "number"
|
||||
? (rawBatch as { id: string; index: number })
|
||||
: undefined;
|
||||
const failedRun = await deps.engine.createFailedTaskRun({
|
||||
friendlyId: args.friendlyId,
|
||||
environment: env,
|
||||
taskIdentifier: String(snapshot.taskIdentifier ?? ""),
|
||||
payload: typeof snapshot.payload === "string" ? snapshot.payload : undefined,
|
||||
payloadType: typeof snapshot.payloadType === "string" ? snapshot.payloadType : undefined,
|
||||
error: {
|
||||
type: "STRING_ERROR",
|
||||
raw: `Mollifier drainer terminal failure: ${args.reason}`,
|
||||
},
|
||||
parentTaskRunId:
|
||||
typeof snapshot.parentTaskRunId === "string" ? snapshot.parentTaskRunId : undefined,
|
||||
rootTaskRunId: typeof snapshot.rootTaskRunId === "string" ? snapshot.rootTaskRunId : undefined,
|
||||
depth: typeof snapshot.depth === "number" ? snapshot.depth : 0,
|
||||
resumeParentOnCompletion: snapshot.resumeParentOnCompletion === true,
|
||||
batch,
|
||||
traceId: typeof snapshot.traceId === "string" ? snapshot.traceId : undefined,
|
||||
spanId: typeof snapshot.spanId === "string" ? snapshot.spanId : undefined,
|
||||
taskEventStore:
|
||||
typeof snapshot.taskEventStore === "string" ? snapshot.taskEventStore : undefined,
|
||||
queue: typeof snapshot.queue === "string" ? snapshot.queue : undefined,
|
||||
lockedQueueId: typeof snapshot.lockedQueueId === "string" ? snapshot.lockedQueueId : undefined,
|
||||
emitRunFailedEvent: false,
|
||||
});
|
||||
// Alerts side of `runFailed` — the engine emit was suppressed above
|
||||
// so we don't create an orphan trace event; enqueue the alert
|
||||
// directly so customers' ERROR channels still see the failure.
|
||||
// Best-effort, mirroring TriggerFailedTaskService.
|
||||
try {
|
||||
await PerformTaskRunAlertsService.enqueue(failedRun.id);
|
||||
} catch (alertsError) {
|
||||
logger.warn("writeMollifierTerminalFailureRow: alert enqueue failed", {
|
||||
friendlyId: args.friendlyId,
|
||||
error: alertsError instanceof Error ? alertsError.message : String(alertsError),
|
||||
});
|
||||
}
|
||||
return failedRun;
|
||||
}
|
||||
|
||||
// Drainer-side terminal-failure callback. Fires from
|
||||
// `MollifierDrainer.processEntry` BEFORE `buffer.fail()` on any path
|
||||
// where the in-handler write didn't already land — currently the
|
||||
// `cause: "max-attempts-exhausted"` case for retryable PG errors. Writes
|
||||
// the same SYSTEM_FAILURE row the non-retryable handler path writes
|
||||
// inline (via the shared `writeMollifierTerminalFailureRow` helper) so
|
||||
// the customer-visible behaviour is identical regardless of how the
|
||||
// failure was classified.
|
||||
//
|
||||
// Re-throws retryable PG errors so the drainer requeues — buffer.fail()ing
|
||||
// here would still lose the run if PG is genuinely unreachable. Throwing
|
||||
// anything else falls through to buffer.fail to avoid an infinite loop on
|
||||
// a genuinely bad snapshot (the drainer logs it).
|
||||
export function createDrainerTerminalFailureHandler(deps: {
|
||||
engine: RunEngine;
|
||||
prisma: PrismaClientOrTransaction;
|
||||
}): MollifierDrainerTerminalFailureHandler<MollifierSnapshot> {
|
||||
return async (input) => {
|
||||
// The handler's own non-retryable terminal path has already written
|
||||
// the SYSTEM_FAILURE row before it throws non-retryable. Only the
|
||||
// retryable-exhausted path reaches us with no row written yet — gate
|
||||
// on `cause` to avoid double-writing for non-retryable failures.
|
||||
if (input.cause !== "max-attempts-exhausted") return;
|
||||
await startSpan(tracer, "mollifier.drained.terminal_failure", async (span) => {
|
||||
span.setAttribute("mollifier.drained", false);
|
||||
span.setAttribute("mollifier.attempts", input.attempts);
|
||||
span.setAttribute("mollifier.run_friendly_id", input.runId);
|
||||
span.setAttribute("mollifier.terminal_failure_cause", input.cause);
|
||||
span.setAttribute("mollifier.terminal_failure_reason", input.error.message);
|
||||
span.setAttribute("taskRunId", input.runId);
|
||||
await writeMollifierTerminalFailureRow(deps, {
|
||||
friendlyId: input.runId,
|
||||
snapshot: input.payload as Record<string, unknown>,
|
||||
reason: input.error.message,
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
import { reportDrainingCount } from "./mollifierTelemetry.server";
|
||||
|
||||
// How often we ZCARD the draining-tracker set. Each poll is a single
|
||||
// O(1) Redis call, so cadence is bounded by "how fresh do we want the
|
||||
// gauge?" rather than cost. 15s gives a tight-enough window to spot a
|
||||
// brief OOM-induced spike without burning RTTs, and lines up well with
|
||||
// typical Prometheus scrape intervals.
|
||||
const POLL_INTERVAL_MS = 15_000;
|
||||
|
||||
let intervalHandle: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
// Polls `mollifier:draining` cardinality on an interval and feeds the
|
||||
// gauge in `mollifierTelemetry.server.ts`. Started from the drainer
|
||||
// worker bootstrap (alongside `drainer.start()`) so it runs on the same
|
||||
// pods that actually pop/ack entries — observability is colocated with
|
||||
// the lifecycle.
|
||||
//
|
||||
// Idempotent: a second call is a no-op (Remix dev hot-reload re-runs
|
||||
// the bootstrap; the existing interval keeps ticking).
|
||||
export function startMollifierDrainingGauge(
|
||||
opts: {
|
||||
intervalMs?: number;
|
||||
getBuffer?: typeof getMollifierBuffer;
|
||||
} = {}
|
||||
): void {
|
||||
if (intervalHandle !== null) return;
|
||||
|
||||
const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
|
||||
const getBuffer = opts.getBuffer ?? getMollifierBuffer;
|
||||
|
||||
// Fire one poll immediately so the gauge populates before the first
|
||||
// scrape rather than reading 0 for a full interval after boot.
|
||||
const tick = async () => {
|
||||
const buffer = getBuffer();
|
||||
if (!buffer) return;
|
||||
try {
|
||||
const count = await buffer.getDrainingCount();
|
||||
reportDrainingCount(count);
|
||||
} catch (err) {
|
||||
// Transient Redis blip — don't tank the loop, just leave the
|
||||
// gauge at its last-known value. A sustained Redis outage will
|
||||
// surface via the drainer's own alerts long before this gauge
|
||||
// staleness becomes a primary signal.
|
||||
logger.warn("Mollifier draining gauge poll failed; keeping previous value", { err });
|
||||
}
|
||||
};
|
||||
|
||||
void tick();
|
||||
// unref so the interval doesn't keep the process alive past
|
||||
// graceful shutdown — the gauge is best-effort, not a flush boundary.
|
||||
intervalHandle = setInterval(() => {
|
||||
void tick();
|
||||
}, intervalMs);
|
||||
intervalHandle.unref?.();
|
||||
}
|
||||
|
||||
// Test seam. Production code never calls this; lifecycle is implicitly
|
||||
// process-end.
|
||||
export function stopMollifierDrainingGauge(): void {
|
||||
if (intervalHandle === null) return;
|
||||
clearInterval(intervalHandle);
|
||||
intervalHandle = null;
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
import { createRealTripEvaluator } from "./mollifierTripEvaluator.server";
|
||||
import {
|
||||
recordDecision,
|
||||
type DecisionOutcome,
|
||||
type RecordDecisionOptions,
|
||||
} from "./mollifierTelemetry.server";
|
||||
|
||||
// `count` is the fleet-wide fixed-window counter for the env (INCR with a
|
||||
// PEXPIRE armed on the first tick of each window — see
|
||||
// `mollifierEvaluateTrip` in `packages/redis-worker/src/mollifier/buffer.ts`).
|
||||
// All webapp replicas pointing at the same Redis share the key
|
||||
// `mollifier:rate:${envId}`, so the threshold is the fleet-wide ceiling
|
||||
// rather than a per-instance one. At a window boundary an env can briefly
|
||||
// admit up to ~2x threshold across the fleet before tripping (fixed-window
|
||||
// not sliding-window). The tripped marker is refreshed on every overage
|
||||
// call, so a sustained burst holds the divert state until the rate falls
|
||||
// below threshold within a window.
|
||||
export type TripDecision =
|
||||
| { divert: false }
|
||||
| {
|
||||
divert: true;
|
||||
reason: "per_env_rate";
|
||||
count: number;
|
||||
threshold: number;
|
||||
windowMs: number;
|
||||
holdMs: number;
|
||||
};
|
||||
|
||||
export type GateOutcome =
|
||||
| { action: "pass_through" }
|
||||
| { action: "mollify"; decision: Extract<TripDecision, { divert: true }> }
|
||||
| { action: "shadow_log"; decision: Extract<TripDecision, { divert: true }> };
|
||||
|
||||
export type GateInputs = {
|
||||
envId: string;
|
||||
orgId: string;
|
||||
taskId: string;
|
||||
// Org-scoped flag overrides — taken from `Organization.featureFlags` on the
|
||||
// AuthenticatedEnvironment at the call site. The repo-wide `flag()` helper
|
||||
// queries the global `FeatureFlag` table; passing per-org overrides lets the
|
||||
// mollifier opt in a single org without touching the global row, matching
|
||||
// the pattern used by `canAccessAi`, `canAccessPrivateConnections`, and the
|
||||
// compute-template beta gate.
|
||||
orgFeatureFlags: Record<string, unknown> | null;
|
||||
// Trigger options that drive the debounce / OTU / triggerAndWait
|
||||
// bypasses. The mollify path can't
|
||||
// serialise stateful callbacks (debounce), can't safely break OTU's
|
||||
// synchronous-rejection contract, and shouldn't intercept single
|
||||
// triggerAndWait (batchTriggerAndWait still funnels through per item).
|
||||
options?: {
|
||||
debounce?: unknown;
|
||||
oneTimeUseToken?: string;
|
||||
parentTaskRunId?: string;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type TripEvaluator = (inputs: GateInputs) => Promise<TripDecision>;
|
||||
|
||||
// DI seam type for consumers (e.g. triggerTask.server.ts) that inject the
|
||||
// gate at construction time. Deliberately narrower than `evaluateGate`'s
|
||||
// real signature — no `deps` param — because consumers only call it with
|
||||
// inputs and rely on the module-level defaults.
|
||||
export type MollifierEvaluateGate = (inputs: GateInputs) => Promise<GateOutcome>;
|
||||
|
||||
export type GateDependencies = {
|
||||
isMollifierEnabled: () => boolean;
|
||||
isShadowModeOn: () => boolean;
|
||||
resolveOrgFlag: (inputs: GateInputs) => Promise<boolean>;
|
||||
evaluator: TripEvaluator;
|
||||
logShadow: (inputs: GateInputs, decision: Extract<TripDecision, { divert: true }>) => void;
|
||||
logMollified: (inputs: GateInputs, decision: Extract<TripDecision, { divert: true }>) => void;
|
||||
recordDecision: (outcome: DecisionOutcome, opts: RecordDecisionOptions) => void;
|
||||
};
|
||||
|
||||
// `options` is a thunk so env reads happen per-evaluation, not at module load.
|
||||
// Don't "simplify" to a plain object — dynamic config relies on the
|
||||
// gate observing whichever env values are live at trigger time.
|
||||
const defaultEvaluator = createRealTripEvaluator({
|
||||
getBuffer: () => getMollifierBuffer(),
|
||||
options: () => ({
|
||||
windowMs: env.TRIGGER_MOLLIFIER_TRIP_WINDOW_MS,
|
||||
threshold: env.TRIGGER_MOLLIFIER_TRIP_THRESHOLD,
|
||||
holdMs: env.TRIGGER_MOLLIFIER_HOLD_MS,
|
||||
}),
|
||||
});
|
||||
|
||||
function logDivertDecision(
|
||||
message: "mollifier.would_mollify" | "mollifier.mollified",
|
||||
inputs: GateInputs,
|
||||
decision: Extract<TripDecision, { divert: true }>
|
||||
): void {
|
||||
logger.debug(message, {
|
||||
envId: inputs.envId,
|
||||
orgId: inputs.orgId,
|
||||
taskId: inputs.taskId,
|
||||
reason: decision.reason,
|
||||
count: decision.count,
|
||||
threshold: decision.threshold,
|
||||
windowMs: decision.windowMs,
|
||||
holdMs: decision.holdMs,
|
||||
});
|
||||
}
|
||||
|
||||
// Resolve the per-org mollifier flag purely from the in-memory
|
||||
// `Organization.featureFlags` JSON. No DB query — `triggerTask` is the
|
||||
// trigger hot path and the webapp CLAUDE.md forbids adding Prisma calls
|
||||
// there. The fleet-wide kill switch lives in `TRIGGER_MOLLIFIER_ENABLED`; rollout
|
||||
// is per-org via the JSON, matching the pattern used by `canAccessAi`,
|
||||
// `hasComputeAccess`, etc. There is no global `FeatureFlag` table read
|
||||
// in this path by design.
|
||||
export function makeResolveMollifierFlag(): (inputs: GateInputs) => Promise<boolean> {
|
||||
return (inputs) => {
|
||||
const override = inputs.orgFeatureFlags?.[FEATURE_FLAG.mollifierEnabled];
|
||||
if (override !== undefined) {
|
||||
const parsed = FeatureFlagCatalog[FEATURE_FLAG.mollifierEnabled].safeParse(override);
|
||||
if (parsed.success) {
|
||||
return Promise.resolve(parsed.data);
|
||||
}
|
||||
}
|
||||
return Promise.resolve(false);
|
||||
};
|
||||
}
|
||||
|
||||
const resolveMollifierFlag = makeResolveMollifierFlag();
|
||||
|
||||
export const defaultGateDependencies: GateDependencies = {
|
||||
isMollifierEnabled: () => env.TRIGGER_MOLLIFIER_ENABLED === "1",
|
||||
isShadowModeOn: () => env.TRIGGER_MOLLIFIER_SHADOW_MODE === "1",
|
||||
resolveOrgFlag: resolveMollifierFlag,
|
||||
evaluator: defaultEvaluator,
|
||||
logShadow: (inputs, decision) => logDivertDecision("mollifier.would_mollify", inputs, decision),
|
||||
logMollified: (inputs, decision) => logDivertDecision("mollifier.mollified", inputs, decision),
|
||||
recordDecision,
|
||||
};
|
||||
|
||||
export async function evaluateGate(
|
||||
inputs: GateInputs,
|
||||
deps: Partial<GateDependencies> = {}
|
||||
): Promise<GateOutcome> {
|
||||
const d = { ...defaultGateDependencies, ...deps };
|
||||
|
||||
// Resolve the per-org flag up front so every decision below — including
|
||||
// the bypasses — can be labelled enrolled vs not on the
|
||||
// `mollifier.decisions` counter. Fail open: a transient error must not
|
||||
// block triggers. The resolver is purely in-memory (reads
|
||||
// `Organization.featureFlags`); it adds no DB round-trip to the hot path.
|
||||
let orgFlagEnabled: boolean;
|
||||
try {
|
||||
orgFlagEnabled = await d.resolveOrgFlag(inputs);
|
||||
} catch (error) {
|
||||
logger.warn("mollifier.resolve_org_flag_failed", {
|
||||
envId: inputs.envId,
|
||||
orgId: inputs.orgId,
|
||||
taskId: inputs.taskId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
orgFlagEnabled = false;
|
||||
}
|
||||
// Passed to every `recordDecision`. `org` only becomes a label for the
|
||||
// (operationally capped) enrolled cohort — the guard is in
|
||||
// `decisionLabels`, so passing orgId unconditionally here is safe.
|
||||
const labels: RecordDecisionOptions = { enrolled: orgFlagEnabled, orgId: inputs.orgId };
|
||||
|
||||
// Debounce bypass. onDebounced is a closure over webapp state and
|
||||
// can't be snapshotted into the buffer for drainer replay. Skip before the
|
||||
// trip evaluator so debounce traffic is never counted against the rate.
|
||||
if (inputs.options?.debounce) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
// OneTimeUseToken bypass. OTU is a security feature on the PUBLIC_JWT
|
||||
// auth path; its synchronous-rejection contract is materially worse to
|
||||
// break than the idempotency-key contract.
|
||||
if (inputs.options?.oneTimeUseToken) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
// Single triggerAndWait bypass. batchTriggerAndWait still funnels
|
||||
// through TriggerTaskService.call per item so the dominant burst pattern
|
||||
// remains covered.
|
||||
if (inputs.options?.parentTaskRunId && inputs.options?.resumeParentOnCompletion) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
|
||||
if (!d.isMollifierEnabled()) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
|
||||
const shadowOn = d.isShadowModeOn();
|
||||
|
||||
if (!orgFlagEnabled && !shadowOn) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
|
||||
// Fail open on evaluator errors too. The default `createRealTripEvaluator`
|
||||
// catches its own errors and returns `{ divert: false }`, but injected or
|
||||
// future evaluators may not — keep the contract symmetric with the org
|
||||
// flag resolution above so the trigger hot path can never be broken by a
|
||||
// gate-internal failure.
|
||||
//
|
||||
// Note: the evaluator INCRs the per-env Redis counter (`mollifier:rate:${envId}`)
|
||||
// in *both* shadow-only and flag-on modes — shadow mode is observation-only at
|
||||
// the user-visible level (no diversion), but not Redis-passive. It has to write
|
||||
// because the threshold is computed from a counter, and a counter that doesn't
|
||||
// increment isn't a counter. There's no cross-org bleed: `RuntimeEnvironment`
|
||||
// is 1:1 with `Organization`, so the per-env counter is effectively per-org.
|
||||
let decision: TripDecision;
|
||||
try {
|
||||
decision = await d.evaluator(inputs);
|
||||
} catch (error) {
|
||||
logger.warn("mollifier.evaluator_failed", {
|
||||
envId: inputs.envId,
|
||||
orgId: inputs.orgId,
|
||||
taskId: inputs.taskId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
decision = { divert: false };
|
||||
}
|
||||
if (!decision.divert) {
|
||||
d.recordDecision("pass_through", labels);
|
||||
return { action: "pass_through" };
|
||||
}
|
||||
|
||||
if (orgFlagEnabled) {
|
||||
d.logMollified(inputs, decision);
|
||||
d.recordDecision("mollify", { ...labels, reason: decision.reason });
|
||||
return { action: "mollify", decision };
|
||||
}
|
||||
|
||||
d.logShadow(inputs, decision);
|
||||
d.recordDecision("shadow_log", { ...labels, reason: decision.reason });
|
||||
return { action: "shadow_log", decision };
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { serialiseMollifierSnapshot, type MollifierSnapshot } from "./mollifierSnapshot.server";
|
||||
import type { TripDecision } from "./mollifierGate.server";
|
||||
|
||||
export type MollifyNotice = {
|
||||
code: "mollifier.queued";
|
||||
message: string;
|
||||
docs: string;
|
||||
};
|
||||
|
||||
export type MollifySyntheticResult = {
|
||||
// `id` is the canonical TaskRun primary key derived from `friendlyId`
|
||||
// via `RunId.fromFriendlyId`. Downstream consumers in the trigger
|
||||
// route — notably `saveRequestIdempotency` — index the request-
|
||||
// idempotency cache by this id; without it the cache stores
|
||||
// `undefined` and Prisma's `findFirst({ where: { id: undefined } })`
|
||||
// on retry strips the predicate and returns an arbitrary TaskRun
|
||||
// (potential cross-tenant leak). Always populated.
|
||||
//
|
||||
// `spanId` is the root-span id allocated at gate-accept time and
|
||||
// stored in the snapshot. Callers like the dashboard's Test action
|
||||
// use it to build a `v3RunSpanPath` URL that auto-opens the right
|
||||
// details panel — without it, the buffered run lands on the
|
||||
// run-detail page with no span selected (parity gap with PG runs).
|
||||
run: { id: string; friendlyId: string; spanId: string };
|
||||
error: undefined;
|
||||
// The race-loser path: if accept's SETNX hit an existing
|
||||
// buffered run with the same (env, task, idempotencyKey), the
|
||||
// response echoes the winner's runId with isCached=true. The
|
||||
// mollifier-queued notice is only attached for the happy accept.
|
||||
isCached: boolean;
|
||||
notice?: MollifyNotice;
|
||||
};
|
||||
|
||||
const NOTICE: MollifyNotice = {
|
||||
code: "mollifier.queued",
|
||||
message: "Trigger accepted into burst buffer. Consider batchTrigger for fan-outs of 100+.",
|
||||
docs: "https://trigger.dev/docs/management/tasks/batch-trigger",
|
||||
};
|
||||
|
||||
export async function mollifyTrigger(args: {
|
||||
runFriendlyId: string;
|
||||
environmentId: string;
|
||||
organizationId: string;
|
||||
engineTriggerInput: MollifierSnapshot;
|
||||
decision: Extract<TripDecision, { divert: true }>;
|
||||
buffer: MollifierBuffer;
|
||||
// Optional idempotency context. When both are passed, accept SETNXes
|
||||
// the lookup so the buffered window participates in trigger-time
|
||||
// dedup symmetrically with PG.
|
||||
idempotencyKey?: string;
|
||||
taskIdentifier?: string;
|
||||
}): Promise<MollifySyntheticResult> {
|
||||
const result = await args.buffer.accept({
|
||||
runId: args.runFriendlyId,
|
||||
envId: args.environmentId,
|
||||
orgId: args.organizationId,
|
||||
payload: serialiseMollifierSnapshot(args.engineTriggerInput),
|
||||
idempotencyKey: args.idempotencyKey,
|
||||
taskIdentifier: args.taskIdentifier,
|
||||
});
|
||||
|
||||
if (result.kind === "duplicate_idempotency") {
|
||||
// Race loser. Echo the winner's runId so the SDK's response shape
|
||||
// matches PG-side idempotency cache hits. The winner's spanId isn't
|
||||
// readily available without a second buffer fetch; an empty string
|
||||
// causes `v3RunSpanPath` to omit the `?span=` param, which matches
|
||||
// current behaviour for cached PG responses.
|
||||
return {
|
||||
run: {
|
||||
id: RunId.fromFriendlyId(result.existingRunId),
|
||||
friendlyId: result.existingRunId,
|
||||
spanId: "",
|
||||
},
|
||||
error: undefined,
|
||||
isCached: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Both "accepted" and "duplicate_run_id" produce the same customer-
|
||||
// visible response: a buffered-trigger acknowledgement. The duplicate
|
||||
// runId case is unreachable in practice (runIds are server-generated
|
||||
// and unique) but is silently idempotent at the buffer layer either way.
|
||||
const rawSpanId = args.engineTriggerInput.spanId;
|
||||
const spanId = typeof rawSpanId === "string" ? rawSpanId : "";
|
||||
return {
|
||||
run: {
|
||||
id: RunId.fromFriendlyId(args.runFriendlyId),
|
||||
friendlyId: args.runFriendlyId,
|
||||
spanId,
|
||||
},
|
||||
error: undefined,
|
||||
isCached: false,
|
||||
notice: NOTICE,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { serialiseSnapshot, deserialiseSnapshot } from "@trigger.dev/redis-worker";
|
||||
|
||||
// MollifierSnapshot is the JSON-serialisable shape of the input that would be
|
||||
// passed to engine.trigger(). The drainer deserialises and replays it.
|
||||
// Kept as Record<string, unknown> at this layer — the engine.trigger call site
|
||||
// casts it to the engine's typed input. This keeps the mollifier subdirectory
|
||||
// from depending on @internal/run-engine internals.
|
||||
export type MollifierSnapshot = Record<string, unknown>;
|
||||
|
||||
export function serialiseMollifierSnapshot(input: MollifierSnapshot): string {
|
||||
return serialiseSnapshot(input);
|
||||
}
|
||||
|
||||
export function deserialiseMollifierSnapshot(serialised: string): MollifierSnapshot {
|
||||
return deserialiseSnapshot<MollifierSnapshot>(serialised);
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { logger as defaultLogger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
import { type StaleSweepStateStore } from "./mollifierStaleSweepState.server";
|
||||
import {
|
||||
recordStaleEntry as defaultRecordStaleEntry,
|
||||
reportStaleEntrySnapshot as defaultReportStaleEntrySnapshot,
|
||||
} from "./mollifierTelemetry.server";
|
||||
|
||||
// One pass of the sweep scans a bounded slice of orgs from the buffer's
|
||||
// queue LIST, identified by a durable cursor in Redis. Per-env entry
|
||||
// scan is also bounded so a single pathological env can't extend the
|
||||
// pass.
|
||||
const DEFAULT_MAX_ENTRIES_PER_ENV = 1000;
|
||||
// Max orgs visited per tick. Together with `maxEntriesPerEnv` this
|
||||
// caps Redis traffic per pass. One "cycle" (visiting every org once)
|
||||
// takes `ceil(N_orgs / cap)` ticks, after which the cursor wraps and a
|
||||
// fresh org list is taken.
|
||||
const DEFAULT_MAX_ORGS_PER_PASS = 100;
|
||||
|
||||
export type StaleSweepConfig = {
|
||||
// Entries whose dwell exceeds this threshold are flagged stale. Set
|
||||
// it well below `entryTtlSeconds * 1000` so ops have lead time before
|
||||
// TTL-induced silent loss; the default (half of entryTtlSeconds)
|
||||
// matches the cadence in the plan doc.
|
||||
staleThresholdMs: number;
|
||||
maxEntriesPerEnv?: number;
|
||||
// Hard cap on orgs visited per tick. Bounds the per-pass Redis traffic
|
||||
// and wall-time. Default 100 — at typical fleet sizes one or two
|
||||
// ticks cover everyone; under incident-scale fan-out a full cycle
|
||||
// takes a handful of ticks (~minutes) which is still well below the
|
||||
// staleness signal latency that ops cares about.
|
||||
maxOrgsPerPass?: number;
|
||||
};
|
||||
|
||||
export type StaleSweepDeps = {
|
||||
getBuffer?: () => MollifierBuffer | null;
|
||||
// Durable cursor + per-env counts hash. Required: the sweep is
|
||||
// useless without persistent state across ticks. The webapp wires up
|
||||
// a real `MollifierStaleSweepState`; tests pass one constructed
|
||||
// against the test container.
|
||||
state: StaleSweepStateStore;
|
||||
// No `envId` arg — `envId` is a high-cardinality metric attribute and
|
||||
// is intentionally not emitted as a metric label. The structured warn
|
||||
// log below carries envId for forensic drill-down.
|
||||
recordStaleEntry?: () => void;
|
||||
reportStaleEntrySnapshot?: (snapshot: Map<string, number>) => void;
|
||||
logger?: { warn: (message: string, fields: Record<string, unknown>) => void };
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
export type StaleSweepResult = {
|
||||
orgsScanned: number;
|
||||
envsScanned: number;
|
||||
entriesScanned: number;
|
||||
staleCount: number;
|
||||
};
|
||||
|
||||
// Walks a bounded slice of `orgs → envs → entries`, emitting an OTel
|
||||
// counter tick and a structured warning log for each buffer entry whose
|
||||
// dwell exceeds the stale threshold. Read-only on the buffer's own
|
||||
// state; writes only to the sweep's three dedicated keys
|
||||
// (`mollifier:stale_sweep:*`). The sweep does NOT remove or salvage
|
||||
// buffer entries; that decision is deferred to a separate retention-
|
||||
// policy change. The signal here exists so ops sees the drainer falling
|
||||
// behind well before TTL-induced loss kicks in.
|
||||
//
|
||||
// Sharding contract:
|
||||
// - Cursor starts at 0. On cursor=0 the org list is refreshed by
|
||||
// snapshotting `buffer.listOrgs()` into the durable LIST — that is
|
||||
// the cycle's frozen view of orgs to visit.
|
||||
// - Each tick consumes up to `maxOrgsPerPass` orgs from the LIST,
|
||||
// advances the cursor, and persists.
|
||||
// - When the cursor reaches the end of the LIST it wraps to 0; the next
|
||||
// tick rebuilds the org list, capturing any orgs that joined the
|
||||
// buffer mid-cycle.
|
||||
// - The per-env counts HASH carries over across ticks: an env visited
|
||||
// on tick N and not revisited until tick N+M keeps its last-known
|
||||
// stale count in the gauge for that window. This is the price of
|
||||
// sharding — accepted because the alternative (re-scan everything
|
||||
// every tick) does not bound work.
|
||||
export async function runStaleSweepOnce(
|
||||
config: StaleSweepConfig,
|
||||
deps: StaleSweepDeps
|
||||
): Promise<StaleSweepResult> {
|
||||
const getBuffer = deps.getBuffer ?? getMollifierBuffer;
|
||||
const recordStale = deps.recordStaleEntry ?? defaultRecordStaleEntry;
|
||||
const reportSnapshot = deps.reportStaleEntrySnapshot ?? defaultReportStaleEntrySnapshot;
|
||||
const log = deps.logger ?? defaultLogger;
|
||||
const now = (deps.now ?? Date.now)();
|
||||
const maxEntries = config.maxEntriesPerEnv ?? DEFAULT_MAX_ENTRIES_PER_ENV;
|
||||
const maxOrgsPerPass = config.maxOrgsPerPass ?? DEFAULT_MAX_ORGS_PER_PASS;
|
||||
|
||||
const buffer = getBuffer();
|
||||
if (!buffer) {
|
||||
// Replace any previous snapshot with empty so a previously-paging
|
||||
// env doesn't stay latched if mollifier is turned off mid-flight.
|
||||
// Also clear the durable state so a re-enable starts from a clean
|
||||
// slate instead of resuming on a stale cursor.
|
||||
await deps.state.clearAll();
|
||||
reportSnapshot(new Map());
|
||||
return { orgsScanned: 0, envsScanned: 0, entriesScanned: 0, staleCount: 0 };
|
||||
}
|
||||
|
||||
let cursor = await deps.state.readCursor();
|
||||
if (cursor === 0) {
|
||||
// Fresh cycle — capture the current set of orgs into the frozen
|
||||
// LIST. Any orgs that join after this snapshot wait until the next
|
||||
// cycle to be visited. Acceptable for an observational sweep; the
|
||||
// staleness signal would only fire on entries that have been
|
||||
// dwelling for `staleThresholdMs` anyway, so they're not new.
|
||||
const orgs = await buffer.listOrgs();
|
||||
await deps.state.rebuildOrgList(orgs);
|
||||
}
|
||||
|
||||
const { orgs: slice, total } = await deps.state.readOrgListSlice(cursor, maxOrgsPerPass);
|
||||
|
||||
let envsScanned = 0;
|
||||
let entriesScanned = 0;
|
||||
let staleCount = 0;
|
||||
|
||||
for (const orgId of slice) {
|
||||
const envs = await buffer.listEnvsForOrg(orgId);
|
||||
for (const envId of envs) {
|
||||
envsScanned += 1;
|
||||
let envStale = 0;
|
||||
const entries = await buffer.listEntriesForEnv(envId, maxEntries);
|
||||
for (const entry of entries) {
|
||||
entriesScanned += 1;
|
||||
const dwellMs = now - entry.createdAt.getTime();
|
||||
if (dwellMs > config.staleThresholdMs) {
|
||||
recordStale();
|
||||
log.warn("mollifier.stale_entry", {
|
||||
runId: entry.runId,
|
||||
envId,
|
||||
orgId,
|
||||
dwellMs,
|
||||
staleThresholdMs: config.staleThresholdMs,
|
||||
});
|
||||
envStale += 1;
|
||||
}
|
||||
}
|
||||
// Persist the per-env count to the durable hash. HSET when stale
|
||||
// > 0, HDEL when it dropped back to zero — the hash is the source
|
||||
// of truth for the gauge snapshot below.
|
||||
await deps.state.setEnvStaleCount(envId, envStale);
|
||||
// Track that this env was visited during the current cycle. The
|
||||
// reconcile step at cycle wrap uses this to HDEL counts hash
|
||||
// entries for envs that fully drained mid-cycle (they disappear
|
||||
// from listEnvsForOrg, so the inner loop above never reaches them
|
||||
// and never HDELs their hash field — without reconcile the gauge
|
||||
// would stay elevated forever).
|
||||
await deps.state.markEnvVisited(envId);
|
||||
staleCount += envStale;
|
||||
}
|
||||
}
|
||||
|
||||
// Advance the cursor. If the slice consumed the end of the LIST, wrap
|
||||
// to 0 so the next tick rebuilds the org list and starts a new cycle.
|
||||
const advanced = cursor + slice.length;
|
||||
const wrapped = advanced >= total;
|
||||
const newCursor = wrapped ? 0 : advanced;
|
||||
await deps.state.writeCursor(newCursor);
|
||||
|
||||
if (wrapped) {
|
||||
// Cycle ended. HDEL any env still in the counts hash that didn't
|
||||
// appear in any tick of the just-completed cycle — these are envs
|
||||
// that fully drained from the buffer mid-cycle and would otherwise
|
||||
// hold their stale gauge value forever. Also DELs the visited set
|
||||
// so the next cycle starts clean.
|
||||
await deps.state.reconcileVisited();
|
||||
}
|
||||
|
||||
// Emit the snapshot from the durable hash, which carries values for
|
||||
// envs visited in earlier ticks too. This is what makes the gauge
|
||||
// stable across ticks (and across webapp restarts).
|
||||
const snapshot = await deps.state.readAllEnvStaleCounts();
|
||||
reportSnapshot(snapshot);
|
||||
|
||||
return { orgsScanned: slice.length, envsScanned, entriesScanned, staleCount };
|
||||
}
|
||||
|
||||
export type StaleSweepIntervalHandle = {
|
||||
stop: () => Promise<void>;
|
||||
};
|
||||
|
||||
// Production wrapper: schedule `runStaleSweepOnce` on a fixed interval.
|
||||
// One pass at a time — if a sweep is still running when the timer fires
|
||||
// the next tick is skipped (a backed-up Redis would otherwise queue
|
||||
// overlapping sweeps that all log the same stale entries).
|
||||
export function startStaleSweepInterval(
|
||||
config: StaleSweepConfig & { intervalMs: number },
|
||||
deps: StaleSweepDeps
|
||||
): StaleSweepIntervalHandle {
|
||||
let stopped = false;
|
||||
let inFlight = false;
|
||||
// Tracks the current tick so `stop()` can await it before closing the
|
||||
// state's Redis client. Without this, a tick that's already past the
|
||||
// `stopped` guard at entry would continue making `state.*` calls
|
||||
// against an ioredis client that `stop()` has already `quit()`ed,
|
||||
// raising errors that the tick's own try/catch then logs as
|
||||
// `mollifier.stale_sweep.failed` warnings — spurious noise on every
|
||||
// graceful shutdown.
|
||||
let currentTick: Promise<void> | null = null;
|
||||
|
||||
const tick = async () => {
|
||||
if (stopped || inFlight) return;
|
||||
inFlight = true;
|
||||
const run = (async () => {
|
||||
try {
|
||||
await runStaleSweepOnce(config, deps);
|
||||
} catch (err) {
|
||||
const log = deps.logger ?? defaultLogger;
|
||||
log.warn("mollifier.stale_sweep.failed", {
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
inFlight = false;
|
||||
currentTick = null;
|
||||
}
|
||||
})();
|
||||
currentTick = run;
|
||||
await run;
|
||||
};
|
||||
|
||||
const timer = setInterval(() => {
|
||||
void tick();
|
||||
}, config.intervalMs);
|
||||
|
||||
return {
|
||||
stop: async () => {
|
||||
stopped = true;
|
||||
clearInterval(timer);
|
||||
// Drain any tick that started before `stopped` flipped. Its
|
||||
// `state.*` calls must land before we close the Redis client.
|
||||
if (currentTick) {
|
||||
try {
|
||||
await currentTick;
|
||||
} catch {
|
||||
// tick has its own catch — this await is just to ensure
|
||||
// ordering, not to surface errors that have already been
|
||||
// logged inside the tick.
|
||||
}
|
||||
}
|
||||
// Close the state's underlying resource. The `close()` method is
|
||||
// part of the `StaleSweepStateStore` contract — production's
|
||||
// `MollifierStaleSweepState` shuts down its ioredis client; fake
|
||||
// test states implement a no-op.
|
||||
await deps.state.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
|
||||
// Durable per-tick state for the sharded stale sweep. Four Redis keys,
|
||||
// all in the `mollifier:` namespace alongside the buffer's own state:
|
||||
//
|
||||
// mollifier:stale_sweep:cursor STRING next position in org_list (0 = fresh cycle)
|
||||
// mollifier:stale_sweep:org_list LIST org IDs frozen at the start of the cycle
|
||||
// mollifier:stale_sweep:counts HASH envId -> last-known stale count
|
||||
// mollifier:stale_sweep:visited SET envIds visited during the current cycle
|
||||
//
|
||||
// The state survives webapp restarts: a restarted process picks up the
|
||||
// cursor where the previous one left off and re-emits the last-known
|
||||
// gauge values immediately, rather than blinking to zero until the next
|
||||
// cycle visits each env.
|
||||
//
|
||||
// The `visited` set exists to GC the `counts` hash at cycle wrap: an env
|
||||
// that drains completely between sweep ticks disappears from
|
||||
// `buffer.listEnvsForOrg`, so the sweep's inner loop never revisits it
|
||||
// and never HDELs its counts entry. Without the visited-set GC the
|
||||
// counts hash retains the env's last-known stale count forever and the
|
||||
// gauge stays permanently elevated. At cursor wrap we diff the hash
|
||||
// against the cycle's visited set and HDEL the difference.
|
||||
//
|
||||
// Storage is owned by this class rather than added to MollifierBuffer
|
||||
// because the keys are sweep-internal — the buffer abstracts the
|
||||
// drainer/queue state, this abstracts sweep state. They share a
|
||||
// namespace prefix but no API surface.
|
||||
|
||||
export interface StaleSweepStateStore {
|
||||
readCursor(): Promise<number>;
|
||||
writeCursor(value: number): Promise<void>;
|
||||
/** Replaces the cycle's frozen org_list. Called at cursor=0. */
|
||||
rebuildOrgList(orgs: string[]): Promise<void>;
|
||||
/** Returns up to `count` org IDs starting at `start`, plus the LIST's total length. */
|
||||
readOrgListSlice(start: number, count: number): Promise<{ orgs: string[]; total: number }>;
|
||||
/** HSET when count > 0, HDEL when count === 0 (so the snapshot reflects current truth). */
|
||||
setEnvStaleCount(envId: string, count: number): Promise<void>;
|
||||
readAllEnvStaleCounts(): Promise<Map<string, number>>;
|
||||
/** SADD `envId` to the current cycle's visited set. Called once per env scanned per tick. */
|
||||
markEnvVisited(envId: string): Promise<void>;
|
||||
/**
|
||||
* HDEL every env in the counts hash that is NOT in the visited set, then
|
||||
* DEL the visited set. Called when the cursor wraps (cycle ends) so
|
||||
* envs that fully drained mid-cycle get cleaned out of the gauge.
|
||||
*/
|
||||
reconcileVisited(): Promise<void>;
|
||||
clearAll(): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
const CURSOR_KEY = "mollifier:stale_sweep:cursor";
|
||||
const ORG_LIST_KEY = "mollifier:stale_sweep:org_list";
|
||||
const COUNTS_KEY = "mollifier:stale_sweep:counts";
|
||||
const VISITED_KEY = "mollifier:stale_sweep:visited";
|
||||
|
||||
export class MollifierStaleSweepState implements StaleSweepStateStore {
|
||||
private readonly redis: Redis;
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(options: {
|
||||
redisOptions: RedisOptions;
|
||||
logger?: Logger;
|
||||
maxRetriesPerRequest?: number;
|
||||
}) {
|
||||
this.logger = options.logger ?? new Logger("MollifierStaleSweepState", "debug");
|
||||
this.redis = createRedisClient(
|
||||
{ ...options.redisOptions, maxRetriesPerRequest: options.maxRetriesPerRequest ?? 20 },
|
||||
{
|
||||
onError: (error) => {
|
||||
this.logger.error("MollifierStaleSweepState redis client error:", { error });
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async readCursor(): Promise<number> {
|
||||
const raw = await this.redis.get(CURSOR_KEY);
|
||||
if (raw === null) return 0;
|
||||
const n = Number.parseInt(raw, 10);
|
||||
return Number.isFinite(n) && n >= 0 ? n : 0;
|
||||
}
|
||||
|
||||
async writeCursor(value: number): Promise<void> {
|
||||
await this.redis.set(CURSOR_KEY, String(value));
|
||||
}
|
||||
|
||||
async rebuildOrgList(orgs: string[]): Promise<void> {
|
||||
// DEL + RPUSH in a pipeline — close enough to atomic for an
|
||||
// observational sweep (the inFlight guard at startStaleSweepInterval
|
||||
// serialises sweep passes; nothing else writes these keys).
|
||||
const pipeline = this.redis.pipeline();
|
||||
pipeline.del(ORG_LIST_KEY);
|
||||
if (orgs.length > 0) {
|
||||
pipeline.rpush(ORG_LIST_KEY, ...orgs);
|
||||
}
|
||||
await pipeline.exec();
|
||||
}
|
||||
|
||||
async readOrgListSlice(start: number, count: number): Promise<{ orgs: string[]; total: number }> {
|
||||
const pipeline = this.redis.pipeline();
|
||||
pipeline.lrange(ORG_LIST_KEY, start, start + count - 1);
|
||||
pipeline.llen(ORG_LIST_KEY);
|
||||
const results = await pipeline.exec();
|
||||
// `pipeline.exec()` returning null is the abort-on-broken-pipe path.
|
||||
// Surface it as a thrown error — the previous `return { orgs: [], total: 0 }`
|
||||
// looked indistinguishable from a genuinely empty org list to the
|
||||
// caller (`runStaleSweepOnce`), which then wrote cursor=0, reconciled
|
||||
// visited envs against the empty result, and cleared the stale-entry
|
||||
// gauge. That hid real Redis problems and silenced the alerts the
|
||||
// sweep exists to raise.
|
||||
if (!results) {
|
||||
throw new Error("MollifierStaleSweepState.readOrgListSlice: pipeline.exec returned null");
|
||||
}
|
||||
const [lrangeErr, lrangeRes] = results[0] as [Error | null, string[] | null];
|
||||
const [llenErr, llenRes] = results[1] as [Error | null, number | null];
|
||||
if (lrangeErr || llenErr) {
|
||||
this.logger.error("MollifierStaleSweepState.readOrgListSlice failed", {
|
||||
lrangeErr: lrangeErr?.message,
|
||||
llenErr: llenErr?.message,
|
||||
});
|
||||
// Same reasoning as the null-result path above — propagate the
|
||||
// failure so the sweep's interval wrapper records a failed cycle
|
||||
// and the durable cursor / counts hash stay untouched.
|
||||
throw lrangeErr ?? llenErr ?? new Error("MollifierStaleSweepState.readOrgListSlice failed");
|
||||
}
|
||||
return { orgs: lrangeRes ?? [], total: llenRes ?? 0 };
|
||||
}
|
||||
|
||||
async setEnvStaleCount(envId: string, count: number): Promise<void> {
|
||||
if (count > 0) {
|
||||
await this.redis.hset(COUNTS_KEY, envId, String(count));
|
||||
} else {
|
||||
await this.redis.hdel(COUNTS_KEY, envId);
|
||||
}
|
||||
}
|
||||
|
||||
async readAllEnvStaleCounts(): Promise<Map<string, number>> {
|
||||
const raw = await this.redis.hgetall(COUNTS_KEY);
|
||||
const out = new Map<string, number>();
|
||||
for (const [envId, value] of Object.entries(raw)) {
|
||||
const n = Number.parseInt(value, 10);
|
||||
if (Number.isFinite(n)) out.set(envId, n);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async markEnvVisited(envId: string): Promise<void> {
|
||||
await this.redis.sadd(VISITED_KEY, envId);
|
||||
}
|
||||
|
||||
async reconcileVisited(): Promise<void> {
|
||||
// HKEYS + SMEMBERS in a pipeline, then HDEL the difference locally.
|
||||
// For typical fleet sizes (counts and visited both bounded by the
|
||||
// count of buffered envs) this is well within a single RTT plus one
|
||||
// small HDEL.
|
||||
const pipeline = this.redis.pipeline();
|
||||
pipeline.hkeys(COUNTS_KEY);
|
||||
pipeline.smembers(VISITED_KEY);
|
||||
const results = await pipeline.exec();
|
||||
if (!results) return;
|
||||
const [hkeysErr, hkeysRes] = results[0] as [Error | null, string[] | null];
|
||||
const [smembersErr, smembersRes] = results[1] as [Error | null, string[] | null];
|
||||
if (hkeysErr || smembersErr) {
|
||||
this.logger.error("MollifierStaleSweepState.reconcileVisited failed", {
|
||||
hkeysErr: hkeysErr?.message,
|
||||
smembersErr: smembersErr?.message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const hashEnvs = hkeysRes ?? [];
|
||||
const visited = new Set(smembersRes ?? []);
|
||||
const orphans = hashEnvs.filter((envId) => !visited.has(envId));
|
||||
const cleanup = this.redis.pipeline();
|
||||
if (orphans.length > 0) {
|
||||
cleanup.hdel(COUNTS_KEY, ...orphans);
|
||||
}
|
||||
cleanup.del(VISITED_KEY);
|
||||
await cleanup.exec();
|
||||
}
|
||||
|
||||
async clearAll(): Promise<void> {
|
||||
await this.redis.del(CURSOR_KEY, ORG_LIST_KEY, COUNTS_KEY, VISITED_KEY);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.redis.quit();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
|
||||
const meter = getMeter("mollifier");
|
||||
|
||||
export const mollifierDecisionsCounter = meter.createCounter("mollifier.decisions", {
|
||||
description: "Count of mollifier gate decisions by outcome",
|
||||
});
|
||||
|
||||
export type DecisionOutcome = "pass_through" | "shadow_log" | "mollify";
|
||||
export type DecisionReason = "per_env_rate";
|
||||
|
||||
export type RecordDecisionOptions = {
|
||||
reason?: DecisionReason;
|
||||
// Whether the org has the per-org mollifier flag enabled. Emitted as the
|
||||
// bounded `enrolled` label so we can see how often enrolled orgs pass
|
||||
// through instead of mollifying — the whole point of this instrumentation.
|
||||
enrolled: boolean;
|
||||
// Org id, attached as the `org` label ONLY when `enrolled` is true. The
|
||||
// enrolled cohort is capped operationally (<= 10 orgs), so this stays
|
||||
// low-cardinality. It must NEVER be attached for non-enrolled orgs — that
|
||||
// would fan the metric out across every org id in production (unbounded;
|
||||
// the same high-cardinality ban that keeps envId/orgId off the other
|
||||
// mollifier metrics). The guard lives in `decisionLabels`, so callers can
|
||||
// pass orgId unconditionally.
|
||||
orgId?: string;
|
||||
};
|
||||
|
||||
// Pure: builds the metric label set for a gate decision. Extracted from
|
||||
// `recordDecision` so the org-only-when-enrolled cardinality guard is
|
||||
// unit-testable without standing up an OTel meter.
|
||||
export function decisionLabels(
|
||||
outcome: DecisionOutcome,
|
||||
opts: RecordDecisionOptions
|
||||
): Record<string, string> {
|
||||
return {
|
||||
outcome,
|
||||
enrolled: opts.enrolled ? "true" : "false",
|
||||
...(opts.reason ? { reason: opts.reason } : {}),
|
||||
...(opts.enrolled && opts.orgId ? { org: opts.orgId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function recordDecision(outcome: DecisionOutcome, opts: RecordDecisionOptions): void {
|
||||
mollifierDecisionsCounter.add(1, decisionLabels(outcome, opts));
|
||||
}
|
||||
|
||||
// Counts subscriptions hitting `/realtime/v1/runs/<id>` for a run that
|
||||
// lives only in the mollifier buffer (no PG row yet). The route opens
|
||||
// the Electric stream anyway so the eventual drainer-INSERT propagates
|
||||
// to the client; this counter is the signal of how often customers
|
||||
// subscribe inside the buffered window.
|
||||
export const realtimeBufferedSubscriptionsCounter = meter.createCounter(
|
||||
"mollifier.realtime_subscriptions.buffered",
|
||||
{
|
||||
description:
|
||||
"Realtime subscriptions opened against a runId that exists only in the mollifier buffer",
|
||||
}
|
||||
);
|
||||
|
||||
// No `envId` attribute — `envId` is a banned high-cardinality metric
|
||||
// label per the repo's OTel rules. The structured warn log emitted
|
||||
// alongside the counter tick (in `mollifierStaleSweep.server.ts`)
|
||||
// carries the envId / orgId / runId for forensic drill-down; the
|
||||
// metric stays an aggregate.
|
||||
export function recordRealtimeBufferedSubscription(): void {
|
||||
realtimeBufferedSubscriptionsCounter.add(1);
|
||||
}
|
||||
|
||||
// Counts buffer entries that have been waiting in the queue ZSET longer
|
||||
// than the configured stale threshold. Useful for historical "stale
|
||||
// events over time" views, but not directly alertable on its own — a
|
||||
// single stuck entry observed by N sweep ticks adds N to the counter,
|
||||
// so `rate()` over an alerting window reflects (entries × ticks), not
|
||||
// "entries that are stale right now".
|
||||
export const staleEntriesCounter = meter.createCounter("mollifier.stale_entries", {
|
||||
description: "Mollifier buffer entries whose dwell exceeds the stale threshold (per sweep pass)",
|
||||
});
|
||||
|
||||
// No `envId` attribute — see comment above.
|
||||
export function recordStaleEntry(): void {
|
||||
staleEntriesCounter.add(1);
|
||||
}
|
||||
|
||||
// Alertable signal: the total count of stale entries observed by the
|
||||
// latest sweep. The sweep snapshots the full picture on each pass so
|
||||
// the gauge drops back to 0 when the drainer catches up instead of
|
||||
// staying latched. Recommended alert:
|
||||
// mollifier_stale_entries_current > 0 for 5m
|
||||
export const staleEntriesGauge = meter.createObservableGauge("mollifier.stale_entries.current", {
|
||||
description:
|
||||
"Buffer entries whose dwell exceeds the stale threshold, as observed by the latest sweep pass",
|
||||
});
|
||||
|
||||
let latestStaleTotal = 0;
|
||||
|
||||
export function reportStaleEntrySnapshot(snapshot: Map<string, number>): void {
|
||||
// Sum across envs. Per-env breakdown is intentionally NOT emitted as
|
||||
// a metric label (high-cardinality); the structured warn log lines
|
||||
// from the sweep carry per-env detail for ops to drill down.
|
||||
let total = 0;
|
||||
for (const count of snapshot.values()) {
|
||||
total += count;
|
||||
}
|
||||
latestStaleTotal = total;
|
||||
}
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
(result) => {
|
||||
result.observe(staleEntriesGauge, latestStaleTotal);
|
||||
},
|
||||
[staleEntriesGauge]
|
||||
);
|
||||
|
||||
// Observability gauge for entries currently in DRAINING state — popped
|
||||
// by the drainer but not yet acked/failed/requeued. Backed by the
|
||||
// `mollifier:draining` ZSET (see `MollifierBuffer.getDrainingCount`)
|
||||
// and polled by the loop in `mollifierDrainingGaugeLoop.server.ts`.
|
||||
//
|
||||
// Useful for:
|
||||
// - "Is anything mid-drain right now?" panels
|
||||
// - Post-crash forensics ("how many entries got stranded by that ECS OOM?")
|
||||
// - Alerting: a sustained non-zero with no drainer progress is a stall
|
||||
//
|
||||
// No `envId` attribute — same high-cardinality constraint as the other
|
||||
// mollifier gauges. The per-entry hash carries env/org for drill-down.
|
||||
export const drainingCountGauge = meter.createObservableGauge("mollifier.draining.current", {
|
||||
description:
|
||||
"Mollifier buffer entries currently in DRAINING state (popped but not yet acked/failed/requeued)",
|
||||
});
|
||||
|
||||
let latestDrainingCount = 0;
|
||||
|
||||
export function reportDrainingCount(count: number): void {
|
||||
latestDrainingCount = count;
|
||||
}
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
(result) => {
|
||||
result.observe(drainingCountGauge, latestDrainingCount);
|
||||
},
|
||||
[drainingCountGauge]
|
||||
);
|
||||
|
||||
// Electric SQL's shape-stream protocol adds a `handle=` query param on
|
||||
// every reconnect after the initial GET. Gating the realtime-buffered
|
||||
// log/counter on its absence keeps the signal at one tick per
|
||||
// subscription instead of one tick per ~20s live-poll iteration —
|
||||
// without it the counter would over-count by the long-poll factor.
|
||||
export function isInitialBufferedSubscriptionRequest(url: string | URL): boolean {
|
||||
const u = typeof url === "string" ? new URL(url) : url;
|
||||
return !u.searchParams.has("handle");
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import type { GateInputs, TripDecision, TripEvaluator } from "./mollifierGate.server";
|
||||
|
||||
export type TripEvaluatorOptions = {
|
||||
windowMs: number;
|
||||
threshold: number;
|
||||
holdMs: number;
|
||||
};
|
||||
|
||||
export type CreateRealTripEvaluatorDeps = {
|
||||
getBuffer: () => MollifierBuffer | null;
|
||||
options: () => TripEvaluatorOptions;
|
||||
};
|
||||
|
||||
export function createRealTripEvaluator(deps: CreateRealTripEvaluatorDeps): TripEvaluator {
|
||||
return async (inputs: GateInputs): Promise<TripDecision> => {
|
||||
const buffer = deps.getBuffer();
|
||||
if (!buffer) return { divert: false };
|
||||
|
||||
const opts = deps.options();
|
||||
|
||||
try {
|
||||
const { tripped, count } = await buffer.evaluateTrip(inputs.envId, opts);
|
||||
if (!tripped) return { divert: false };
|
||||
|
||||
return {
|
||||
divert: true,
|
||||
reason: "per_env_rate",
|
||||
count,
|
||||
threshold: opts.threshold,
|
||||
windowMs: opts.windowMs,
|
||||
holdMs: opts.holdMs,
|
||||
};
|
||||
} catch (err) {
|
||||
// Deliberate: no error counter here. Shadow mode means a silent miss is
|
||||
// harmless — fail-open is the safe direction. The error log + Sentry
|
||||
// capture is sufficient operability while this runs in shadow mode. Revisit
|
||||
// once buffer writes are the primary path and a missed evaluation has cost.
|
||||
logger.error("mollifier trip evaluator: fail-open on error", {
|
||||
envId: inputs.envId,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return { divert: false };
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
import type {
|
||||
BufferEntry,
|
||||
MollifierBuffer,
|
||||
MutateSnapshotResult,
|
||||
SnapshotPatch,
|
||||
} from "@trigger.dev/redis-worker";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import type { PrismaClientOrTransaction, PrismaReplicaClient } from "~/db.server";
|
||||
import { prisma, $replica } from "~/db.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
|
||||
// Wait/retry knobs. Exported for tests.
|
||||
export const DEFAULT_SAFETY_NET_MS = 2_000;
|
||||
// Initial gap between buffer polls; grows by BACKOFF_FACTOR up to
|
||||
// DEFAULT_MAX_POLL_STEP_MS so a slow drain doesn't poll at a tight fixed
|
||||
// cadence for the whole safety-net budget.
|
||||
export const DEFAULT_POLL_STEP_MS = 20;
|
||||
export const DEFAULT_MAX_POLL_STEP_MS = 250;
|
||||
export const DEFAULT_BACKOFF_FACTOR = 1.7;
|
||||
|
||||
export type MutateWithFallbackInput<TResponse> = {
|
||||
runId: string;
|
||||
environmentId: string;
|
||||
organizationId: string;
|
||||
bufferPatch: SnapshotPatch;
|
||||
// Called when a PG row exists (either replica-hit or post-wait writer-hit).
|
||||
// Receives the full TaskRun shape and returns the customer-visible body.
|
||||
pgMutation: (pgRow: TaskRun) => Promise<TResponse>;
|
||||
// Called when the patch landed cleanly on the buffer snapshot. The
|
||||
// drainer will see the patched payload on its next pop. Receives the
|
||||
// pre-mutation snapshot entry (the one fetched for the env auth
|
||||
// check above) so the caller can compute response details that
|
||||
// depend on the prior state — e.g. the tags route needs to dedup
|
||||
// against the existing tags to report an accurate `newTags` count
|
||||
// matching the PG path, without an extra Redis round-trip.
|
||||
// `bufferEntry` is `null` in the rare race where the entry didn't
|
||||
// exist at pre-check time but appeared before `mutateSnapshot`.
|
||||
synthesisedResponse: (ctx: { bufferEntry: BufferEntry | null }) => TResponse | Promise<TResponse>;
|
||||
// Called when the buffer rejected the patch as invalid (e.g. an
|
||||
// `append_tags` patch carrying `maxTags` would exceed the cap). Required
|
||||
// only by callers that send a rejectable patch; the helper throws if the
|
||||
// buffer reports a rejection and no builder was supplied. Receives the
|
||||
// same `bufferEntry` context as `synthesisedResponse` so a rejection
|
||||
// message can reference the prior state if useful.
|
||||
rejectedResponse?: (ctx: { bufferEntry: BufferEntry | null }) => TResponse | Promise<TResponse>;
|
||||
abortSignal?: AbortSignal;
|
||||
// Override defaults for tests.
|
||||
safetyNetMs?: number;
|
||||
pollStepMs?: number;
|
||||
maxPollStepMs?: number;
|
||||
backoffFactor?: number;
|
||||
// Test injection.
|
||||
getBuffer?: () => MollifierBuffer | null;
|
||||
prismaWriter?: PrismaClientOrTransaction;
|
||||
prismaReplica?: PrismaReplicaClient;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
now?: () => number;
|
||||
// Jitter source; defaults to Math.random. Inject `() => 0` for
|
||||
// deterministic poll timing in tests.
|
||||
random?: () => number;
|
||||
};
|
||||
|
||||
export type MutateWithFallbackOutcome<TResponse> =
|
||||
| { kind: "pg"; response: TResponse }
|
||||
| { kind: "snapshot"; response: TResponse }
|
||||
| { kind: "rejected"; response: TResponse }
|
||||
| { kind: "not_found" }
|
||||
| { kind: "timed_out" };
|
||||
|
||||
// PG-first → buffer mutateSnapshot → wait-and-bounce. The
|
||||
// caller decides how to translate the outcome into an HTTP response —
|
||||
// this helper never throws Response objects so it remains route-agnostic
|
||||
// and unit-testable in isolation.
|
||||
export async function mutateWithFallback<TResponse>(
|
||||
input: MutateWithFallbackInput<TResponse>
|
||||
): Promise<MutateWithFallbackOutcome<TResponse>> {
|
||||
const replica = input.prismaReplica ?? $replica;
|
||||
const writer = input.prismaWriter ?? prisma;
|
||||
const buffer = (input.getBuffer ?? getMollifierBuffer)();
|
||||
const sleep = input.sleep ?? defaultSleep;
|
||||
const now = input.now ?? Date.now;
|
||||
|
||||
// Path 1 — PG is already canonical.
|
||||
const replicaRow = await findRunInPg(replica, input.runId, input.environmentId);
|
||||
if (replicaRow) {
|
||||
const response = await input.pgMutation(replicaRow);
|
||||
return { kind: "pg", response };
|
||||
}
|
||||
|
||||
if (!buffer) {
|
||||
// No buffer configured (mollifier disabled or boot-time error). The
|
||||
// pre-PR mutation routes read from the writer directly, so a freshly-
|
||||
// created PG row was always visible regardless of replication lag.
|
||||
// Now that the read moved to the replica (line 87) for the offload,
|
||||
// a `!buffer` short-circuit would regress: a real PG row + replica
|
||||
// lag would return 404. Mirror the writer-disambiguation block below
|
||||
// (line 148, the buffer-says-not-found path) so degraded mode
|
||||
// (mollifier disabled) still matches pre-PR mutation behaviour.
|
||||
const writerRow = await findRunInPg(writer, input.runId, input.environmentId);
|
||||
if (writerRow) {
|
||||
const response = await input.pgMutation(writerRow);
|
||||
return { kind: "pg", response };
|
||||
}
|
||||
return { kind: "not_found" };
|
||||
}
|
||||
|
||||
// Env-scoped authorization for the buffer path. The replica/writer
|
||||
// lookups above are already env-scoped via findRunInPg; this closes
|
||||
// the same gap on the buffer side so a caller authed in env A can't
|
||||
// mutate a buffered run that belongs to env B (or a different org)
|
||||
// by guessing its friendlyId. Non-atomic w.r.t. the mutateSnapshot
|
||||
// call below, but the TOCTOU is benign: runIds are globally unique,
|
||||
// so a cross-env entry can't suddenly appear after a same-env check.
|
||||
// A genuinely-missing entry (entry === null) falls through and is
|
||||
// handled by the existing not_found / writer-recovery path below.
|
||||
const entryForAuth = await buffer.getEntry(input.runId);
|
||||
if (
|
||||
entryForAuth &&
|
||||
(entryForAuth.envId !== input.environmentId || entryForAuth.orgId !== input.organizationId)
|
||||
) {
|
||||
// Hide existence on env mismatch: return not_found, same shape as
|
||||
// a true miss, rather than 403 which would leak that the runId
|
||||
// exists in some other env.
|
||||
return { kind: "not_found" };
|
||||
}
|
||||
|
||||
// Path 2 — buffer snapshot mutation.
|
||||
const result: MutateSnapshotResult = await buffer.mutateSnapshot(input.runId, input.bufferPatch);
|
||||
|
||||
if (result === "applied_to_snapshot") {
|
||||
return {
|
||||
kind: "snapshot",
|
||||
response: await input.synthesisedResponse({ bufferEntry: entryForAuth }),
|
||||
};
|
||||
}
|
||||
|
||||
if (result === "limit_exceeded") {
|
||||
// The buffer refused the patch (e.g. tag cap). Nothing was written.
|
||||
// Surface the caller's rejection body; a missing builder means the
|
||||
// caller sent a rejectable patch without handling the rejection.
|
||||
if (!input.rejectedResponse) {
|
||||
throw new Error(
|
||||
"mutateWithFallback: buffer returned 'limit_exceeded' but no rejectedResponse was provided"
|
||||
);
|
||||
}
|
||||
return {
|
||||
kind: "rejected",
|
||||
response: await input.rejectedResponse({ bufferEntry: entryForAuth }),
|
||||
};
|
||||
}
|
||||
|
||||
if (result === "not_found") {
|
||||
// Disambiguate a genuine 404 from a replica-lag miss: ask the writer
|
||||
// directly. If the row just appeared post-drain we route through the
|
||||
// PG mutation path.
|
||||
const writerRow = await findRunInPg(writer, input.runId, input.environmentId);
|
||||
if (writerRow) {
|
||||
const response = await input.pgMutation(writerRow);
|
||||
return { kind: "pg", response };
|
||||
}
|
||||
return { kind: "not_found" };
|
||||
}
|
||||
|
||||
// result === "busy" — the entry is mid-handoff (DRAINING) or already
|
||||
// materialised. We do NOT poll the primary for the row to appear: that
|
||||
// piles read load onto the writer at exactly the moment mollifier exists
|
||||
// to shed it. Instead we watch the buffer entry itself (cheap Redis
|
||||
// reads). The drainer writes the PG row BEFORE it acks (sets
|
||||
// `materialised`) or fails (deletes the entry), so the entry's own state
|
||||
// is an authoritative, already-in-Redis signal for "is the row in PG
|
||||
// yet?". Only once it resolves do we touch the primary — exactly once,
|
||||
// for the real mutation.
|
||||
const safetyNetMs = input.safetyNetMs ?? DEFAULT_SAFETY_NET_MS;
|
||||
const maxPollStepMs = input.maxPollStepMs ?? DEFAULT_MAX_POLL_STEP_MS;
|
||||
const backoffFactor = input.backoffFactor ?? DEFAULT_BACKOFF_FACTOR;
|
||||
const random = input.random ?? Math.random;
|
||||
const deadline = now() + safetyNetMs;
|
||||
let step = input.pollStepMs ?? DEFAULT_POLL_STEP_MS;
|
||||
|
||||
while (now() < deadline) {
|
||||
if (input.abortSignal?.aborted) {
|
||||
return { kind: "timed_out" };
|
||||
}
|
||||
|
||||
const entry = await buffer.getEntry(input.runId);
|
||||
// Resolved when the entry is gone (`fail` deleted it after writing a
|
||||
// terminal SYSTEM_FAILURE row) or materialised (`ack` after a
|
||||
// successful trigger / cancel write). In both cases the PG row is now
|
||||
// committed on the primary, so read it once and route through the
|
||||
// canonical PG mutation path.
|
||||
if (entry === null || entry.materialised === true) {
|
||||
const row = await findRunInPg(writer, input.runId, input.environmentId);
|
||||
if (row) {
|
||||
const response = await input.pgMutation(row);
|
||||
return { kind: "pg", response };
|
||||
}
|
||||
// Entry gone with no PG row: the drainer's terminal write itself
|
||||
// failed (PG unreachable). Nothing to mutate.
|
||||
return { kind: "not_found" };
|
||||
}
|
||||
// Still QUEUED (requeued after a retryable drain error) or DRAINING —
|
||||
// the run hasn't reached PG. Back off with jitter so concurrent
|
||||
// waiters on the same draining run don't requery in lockstep.
|
||||
if (now() >= deadline) break;
|
||||
const jittered = step + Math.floor(random() * step);
|
||||
await sleep(jittered);
|
||||
step = Math.min(Math.ceil(step * backoffFactor), maxPollStepMs);
|
||||
}
|
||||
|
||||
logger.warn("mollifier mutate-with-fallback: drainer resolution timed out", {
|
||||
runId: input.runId,
|
||||
safetyNetMs,
|
||||
});
|
||||
return { kind: "timed_out" };
|
||||
}
|
||||
|
||||
async function findRunInPg(
|
||||
client: PrismaClientOrTransaction | PrismaReplicaClient,
|
||||
friendlyId: string,
|
||||
environmentId: string
|
||||
): Promise<TaskRun | null> {
|
||||
return runStore.findRun({ friendlyId, runtimeEnvironmentId: environmentId }, client);
|
||||
}
|
||||
|
||||
function defaultSleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { IdempotencyKeyOptionsSchema } from "@trigger.dev/core/v3/schemas";
|
||||
import type { z } from "zod";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { deserialiseMollifierSnapshot } from "./mollifierSnapshot.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
|
||||
export type ReadFallbackInput = {
|
||||
runId: string;
|
||||
environmentId: string;
|
||||
organizationId: string;
|
||||
};
|
||||
|
||||
export type SyntheticRun = {
|
||||
// Snapshot-derived TaskRun primary key. Used by ReplayTaskRunService
|
||||
// for logging and by callers passing this object where a TaskRun is
|
||||
// expected (cast). Derived deterministically from `friendlyId`.
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
status: "QUEUED" | "FAILED" | "CANCELED";
|
||||
// Set when the customer cancelled the run via the dashboard or API
|
||||
// while it was buffered. The drainer's cancel bifurcation reads this
|
||||
// on next pop and writes a CANCELED PG row directly (skipping
|
||||
// materialisation). Reflected back into the UI by the synthesised
|
||||
// SpanRun so the run-detail page shows the cancelled state even before
|
||||
// the drainer materialises it.
|
||||
cancelledAt: Date | undefined;
|
||||
cancelReason: string | undefined;
|
||||
// Reschedule patch (`set_delay`) writes `delayUntil` into the snapshot.
|
||||
// Surfacing it on SyntheticRun lets the retrieve-run shape reflect the
|
||||
// pending delay before the drainer materialises the PG row.
|
||||
delayUntil: Date | undefined;
|
||||
taskIdentifier: string | undefined;
|
||||
createdAt: Date;
|
||||
|
||||
payload: unknown;
|
||||
payloadType: string | undefined;
|
||||
metadata: unknown;
|
||||
metadataType: string | undefined;
|
||||
// Seed-metadata mirrors what `triggerTask.server.ts` writes into the
|
||||
// snapshot: the original metadataPacket data preserved separately from
|
||||
// any later customer mutations. ReplayTaskRunService uses these to
|
||||
// rebuild the replay's metadata.
|
||||
seedMetadata: string | undefined;
|
||||
seedMetadataType: string | undefined;
|
||||
|
||||
idempotencyKey: string | undefined;
|
||||
// Surfaced for the cached-hit expiration check in IdempotencyKeyConcern.
|
||||
// The PG-resident path enforces this (clears key, allows new run when
|
||||
// expired). For buffered runs the snapshot carries the same field — we
|
||||
// expose it here so the cached-hit branch can apply the same check
|
||||
// rather than indefinitely returning the buffered run's id.
|
||||
idempotencyKeyExpiresAt: Date | undefined;
|
||||
// `{ key, scope }` object form, matching how the SDK serialises and PG
|
||||
// stores it. Previously typed as `string[]` (legacy/incorrect — Prisma
|
||||
// is `Json?` carrying the schema-shaped object). `getUserProvidedIdempotencyKey`
|
||||
// and `extractIdempotencyKeyScope` both parse via the same Zod schema;
|
||||
// they returned `undefined` for the array-shape, which silently
|
||||
// demoted the response to surface the hash instead of the user-
|
||||
// provided key for buffered runs — a contract divergence from
|
||||
// PG-resident runs. See the regression test in `mollifierReadFallback.test.ts`.
|
||||
idempotencyKeyOptions: z.infer<typeof IdempotencyKeyOptionsSchema> | undefined;
|
||||
isTest: boolean;
|
||||
depth: number;
|
||||
ttl: string | undefined;
|
||||
tags: string[];
|
||||
// Mirror of `tags` under the PG field name. ReplayTaskRunService reads
|
||||
// `existingTaskRun.runTags`; both names are kept here so a synthetic
|
||||
// run can be passed wherever the PG-shape `runTags` is expected.
|
||||
runTags: string[];
|
||||
lockedToVersion: string | undefined;
|
||||
resumeParentOnCompletion: boolean;
|
||||
parentTaskRunId: string | undefined;
|
||||
|
||||
// Allocated at gate-accept time and embedded in the snapshot so the run's
|
||||
// trace is continuous from QUEUED-in-buffer through executing post-drain.
|
||||
traceId: string | undefined;
|
||||
spanId: string | undefined;
|
||||
parentSpanId: string | undefined;
|
||||
|
||||
// Replay-relevant fields populated from the engine-trigger snapshot.
|
||||
// ReplayTaskRunService reads each of these from the existing TaskRun;
|
||||
// when the original lives in the buffer we synthesise them here.
|
||||
runtimeEnvironmentId: string | undefined;
|
||||
engine: "V2";
|
||||
workerQueue: string | undefined;
|
||||
region: string | undefined;
|
||||
queue: string | undefined;
|
||||
concurrencyKey: string | undefined;
|
||||
machinePreset: string | undefined;
|
||||
realtimeStreamsVersion: string | undefined;
|
||||
|
||||
// Additional snapshot-sourced fields used when synthesising a SpanRun
|
||||
// for the dashboard's right-side details panel. All optional because
|
||||
// older snapshots may not carry them.
|
||||
maxAttempts: number | undefined;
|
||||
maxDurationInSeconds: number | undefined;
|
||||
replayedFromTaskRunFriendlyId: string | undefined;
|
||||
annotations: unknown;
|
||||
traceContext: unknown;
|
||||
scheduleId: string | undefined;
|
||||
batchId: string | undefined;
|
||||
parentTaskRunFriendlyId: string | undefined;
|
||||
rootTaskRunFriendlyId: string | undefined;
|
||||
|
||||
error?: { code: string; message: string };
|
||||
};
|
||||
|
||||
export type ReadFallbackDeps = {
|
||||
getBuffer?: () => MollifierBuffer | null;
|
||||
};
|
||||
|
||||
function asString(value: unknown): string | undefined {
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) && value.every((v) => typeof v === "string")
|
||||
? (value as string[])
|
||||
: [];
|
||||
}
|
||||
|
||||
function asDate(value: unknown): Date | undefined {
|
||||
const raw = asString(value);
|
||||
if (!raw) return undefined;
|
||||
const parsed = new Date(raw);
|
||||
return Number.isNaN(parsed.getTime()) ? undefined : parsed;
|
||||
}
|
||||
|
||||
// Snapshot ids are written by engine.trigger as INTERNAL ids (cuids); the
|
||||
// SyntheticRun contract exposes friendlyIds. `RunId.toFriendlyId` is
|
||||
// already used for the synthetic run's own id (line 155); reuse it for
|
||||
// parent/root so consumers see the same shape as the PG path.
|
||||
function internalRunIdToFriendlyId(internalId: string | undefined): string | undefined {
|
||||
if (!internalId) return undefined;
|
||||
return RunId.toFriendlyId(internalId);
|
||||
}
|
||||
|
||||
export async function findRunByIdWithMollifierFallback(
|
||||
input: ReadFallbackInput,
|
||||
deps: ReadFallbackDeps = {}
|
||||
): Promise<SyntheticRun | null> {
|
||||
const buffer = (deps.getBuffer ?? getMollifierBuffer)();
|
||||
if (!buffer) return null;
|
||||
|
||||
try {
|
||||
const entry = await buffer.getEntry(input.runId);
|
||||
if (!entry) return null;
|
||||
|
||||
if (entry.envId !== input.environmentId || entry.orgId !== input.organizationId) {
|
||||
logger.warn("mollifier read-fallback auth mismatch", {
|
||||
runId: input.runId,
|
||||
callerEnvId: input.environmentId,
|
||||
callerOrgId: input.organizationId,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const snapshot = deserialiseMollifierSnapshot(entry.payload);
|
||||
// Parse via the canonical schema (`{ key: string, scope: "run" |
|
||||
// "attempt" | "global" }`) rather than the legacy Array.isArray
|
||||
// check. The SDK and Prisma both store this as an object; the array
|
||||
// form never matches, so a buffered run's response previously fell
|
||||
// back to the server-side hash in `getUserProvidedIdempotencyKey`
|
||||
// instead of the customer-supplied key — diverging from how
|
||||
// materialised runs render the same field.
|
||||
const idempotencyKeyOptionsParsed = IdempotencyKeyOptionsSchema.safeParse(
|
||||
snapshot.idempotencyKeyOptions
|
||||
);
|
||||
const idempotencyKeyOptions = idempotencyKeyOptionsParsed.success
|
||||
? idempotencyKeyOptionsParsed.data
|
||||
: undefined;
|
||||
|
||||
const tags = asStringArray(snapshot.tags);
|
||||
const environment =
|
||||
snapshot.environment && typeof snapshot.environment === "object"
|
||||
? (snapshot.environment as Record<string, unknown>)
|
||||
: undefined;
|
||||
|
||||
const cancelledAt = asDate(snapshot.cancelledAt);
|
||||
const cancelReason = asString(snapshot.cancelReason);
|
||||
let status: SyntheticRun["status"] = "QUEUED";
|
||||
if (cancelledAt) {
|
||||
status = "CANCELED";
|
||||
} else if (entry.status === "FAILED") {
|
||||
status = "FAILED";
|
||||
}
|
||||
const delayUntil = asDate(snapshot.delayUntil);
|
||||
|
||||
return {
|
||||
id: RunId.fromFriendlyId(entry.runId),
|
||||
friendlyId: entry.runId,
|
||||
status,
|
||||
cancelledAt,
|
||||
cancelReason,
|
||||
delayUntil,
|
||||
taskIdentifier: asString(snapshot.taskIdentifier),
|
||||
createdAt: entry.createdAt,
|
||||
|
||||
payload: snapshot.payload,
|
||||
payloadType: asString(snapshot.payloadType),
|
||||
metadata: snapshot.metadata,
|
||||
metadataType: asString(snapshot.metadataType),
|
||||
seedMetadata: asString(snapshot.seedMetadata),
|
||||
seedMetadataType: asString(snapshot.seedMetadataType),
|
||||
|
||||
idempotencyKey: asString(snapshot.idempotencyKey),
|
||||
idempotencyKeyExpiresAt: asDate(snapshot.idempotencyKeyExpiresAt),
|
||||
idempotencyKeyOptions,
|
||||
isTest: snapshot.isTest === true,
|
||||
depth: typeof snapshot.depth === "number" ? snapshot.depth : 0,
|
||||
ttl: asString(snapshot.ttl),
|
||||
tags,
|
||||
runTags: tags,
|
||||
lockedToVersion: asString(snapshot.taskVersion),
|
||||
resumeParentOnCompletion: snapshot.resumeParentOnCompletion === true,
|
||||
parentTaskRunId: asString(snapshot.parentTaskRunId),
|
||||
|
||||
traceId: asString(snapshot.traceId),
|
||||
spanId: asString(snapshot.spanId),
|
||||
parentSpanId: asString(snapshot.parentSpanId),
|
||||
|
||||
runtimeEnvironmentId: asString(environment?.id) ?? entry.envId,
|
||||
engine: "V2",
|
||||
workerQueue: asString(snapshot.workerQueue),
|
||||
region: asString(snapshot.region),
|
||||
queue: asString(snapshot.queue),
|
||||
concurrencyKey: asString(snapshot.concurrencyKey),
|
||||
machinePreset: asString(snapshot.machine),
|
||||
realtimeStreamsVersion: asString(snapshot.realtimeStreamsVersion),
|
||||
|
||||
maxAttempts: typeof snapshot.maxAttempts === "number" ? snapshot.maxAttempts : undefined,
|
||||
maxDurationInSeconds:
|
||||
typeof snapshot.maxDurationInSeconds === "number"
|
||||
? snapshot.maxDurationInSeconds
|
||||
: undefined,
|
||||
replayedFromTaskRunFriendlyId: asString(snapshot.replayedFromTaskRunFriendlyId),
|
||||
annotations: snapshot.annotations,
|
||||
traceContext: snapshot.traceContext,
|
||||
scheduleId: asString(snapshot.scheduleId),
|
||||
// The engine.trigger input embeds the batch as `{ id, index }` (see
|
||||
// triggerTask.server.ts #buildEngineTriggerInput), not as a flat
|
||||
// `batchId`. The nested `id` is the batch's internal cuid — the same
|
||||
// value PG stores in `TaskRun.batchId` — so callers reconstruct the
|
||||
// friendly id via `BatchId.toFriendlyId` exactly as the PG path does.
|
||||
batchId: asString((snapshot.batch as { id?: unknown } | undefined)?.id),
|
||||
// The snapshot only carries the INTERNAL parent/root ids
|
||||
// (`parentTaskRunId` / `rootTaskRunId` — what engine.trigger consumes),
|
||||
// not the friendlyIds the SyntheticRun contract expects. Convert
|
||||
// internal → friendly here so consumers don't have to special-case
|
||||
// the buffered path.
|
||||
parentTaskRunFriendlyId: internalRunIdToFriendlyId(asString(snapshot.parentTaskRunId)),
|
||||
rootTaskRunFriendlyId: internalRunIdToFriendlyId(asString(snapshot.rootTaskRunId)),
|
||||
|
||||
error: entry.lastError,
|
||||
};
|
||||
} catch (err) {
|
||||
logger.error("mollifier read-fallback errored — fail-open to null", {
|
||||
runId: input.runId,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import type { PrismaClientOrTransaction, PrismaReplicaClient } from "~/db.server";
|
||||
import { $replica as defaultReplica, prisma as defaultWriter } from "~/db.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { getMollifierBuffer as defaultGetBuffer } from "./mollifierBuffer.server";
|
||||
|
||||
// Discriminated-union resolver used by mutation routes' `findResource`.
|
||||
// The route builder treats a null return from `findResource` as a 404
|
||||
// BEFORE the action handler runs (`apiBuilder.server.ts:321`), so we
|
||||
// must check BOTH the PG canonical store and the mollifier buffer here
|
||||
// — otherwise a buffered run can't be cancelled / mutated even though
|
||||
// the underlying mutateWithFallback flow would handle it correctly.
|
||||
//
|
||||
// (Regression: before extracting this helper the cancel route had
|
||||
// `findResource: async () => null`, which made every cancel 404 before
|
||||
// the action ran. The helper makes the lookup unit-testable.)
|
||||
export type ResolvedRunForMutation =
|
||||
| { source: "pg"; friendlyId: string }
|
||||
| { source: "buffer"; friendlyId: string };
|
||||
|
||||
export type ResolveRunForMutationDeps = {
|
||||
prismaReplica?: PrismaReplicaClient;
|
||||
prismaWriter?: PrismaClientOrTransaction;
|
||||
getBuffer?: () => MollifierBuffer | null;
|
||||
};
|
||||
|
||||
export async function resolveRunForMutation(input: {
|
||||
runParam: string;
|
||||
environmentId: string;
|
||||
organizationId: string;
|
||||
deps?: ResolveRunForMutationDeps;
|
||||
}): Promise<ResolvedRunForMutation | null> {
|
||||
const replica = input.deps?.prismaReplica ?? defaultReplica;
|
||||
const writer = input.deps?.prismaWriter ?? defaultWriter;
|
||||
const getBuffer = input.deps?.getBuffer ?? defaultGetBuffer;
|
||||
|
||||
const pgRun = await runStore.findRun(
|
||||
{ friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId },
|
||||
{ select: { friendlyId: true } },
|
||||
replica
|
||||
);
|
||||
if (pgRun) return { source: "pg", friendlyId: pgRun.friendlyId };
|
||||
|
||||
const buffer = getBuffer();
|
||||
|
||||
if (buffer) {
|
||||
const entry = await buffer.getEntry(input.runParam);
|
||||
if (entry && entry.envId === input.environmentId && entry.orgId === input.organizationId) {
|
||||
return { source: "buffer", friendlyId: input.runParam };
|
||||
}
|
||||
}
|
||||
|
||||
// Replica + buffer both missed. Before declaring "not found" (which the
|
||||
// route builder converts to a hard 404 *before* the action handler runs,
|
||||
// so the downstream `mutateWithFallback` writer-recovery never gets a
|
||||
// chance to fire), do one final probe against the writer. This catches
|
||||
// two cases:
|
||||
// 1. Replica lag on a freshly-created PG row.
|
||||
// 2. A buffered run that materialised in the window between the
|
||||
// replica read and our buffer check (the entry was ack'd and the
|
||||
// hash is mid-grace-TTL but our getEntry returned null due to
|
||||
// lookup-by-friendlyId timing).
|
||||
// Without this, the resolver returns null in degraded states that the
|
||||
// downstream mutateWithFallback flow would otherwise handle correctly.
|
||||
const writerRun = await runStore.findRun(
|
||||
{ friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId },
|
||||
{ select: { friendlyId: true } },
|
||||
writer
|
||||
);
|
||||
if (writerRun) return { source: "pg", friendlyId: writerRun.friendlyId };
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { SyntheticRun } from "./readFallback.server";
|
||||
|
||||
// Buffered runs have no execution data — the drainer hasn't materialised
|
||||
// the PG row and the worker hasn't started. The SDK-facing read routes
|
||||
// still need to return a span/trace shape that satisfies their response
|
||||
// schemas; these helpers build that minimal shape from the buffered
|
||||
// SyntheticRun.
|
||||
//
|
||||
// CANCELED and FAILED are terminal states: a FAILED buffered run is
|
||||
// errored (drainer exhausted retries or the gate rejected it) and must
|
||||
// not signal "still in progress." The flags below mirror
|
||||
// syntheticTrace.server.ts so the SDK contract stays consistent across
|
||||
// the three read paths (spans, trace, dashboard trace presenter).
|
||||
|
||||
function deriveTerminalFlags(status: SyntheticRun["status"]): {
|
||||
isError: boolean;
|
||||
isPartial: boolean;
|
||||
isCancelled: boolean;
|
||||
} {
|
||||
const isCancelled = status === "CANCELED";
|
||||
const isFailed = status === "FAILED";
|
||||
return {
|
||||
isError: isFailed,
|
||||
isPartial: !isCancelled && !isFailed,
|
||||
isCancelled,
|
||||
};
|
||||
}
|
||||
|
||||
// Body for GET /api/v1/runs/:runId/spans/:spanId when the run is buffered
|
||||
// and `:spanId` has already been verified against `buffered.spanId` by the
|
||||
// route. Pure function so the route layer just authenticates, resolves
|
||||
// the run, validates the spanId, and forwards the buffered run here.
|
||||
export function buildSyntheticSpanDetailBody(buffered: SyntheticRun) {
|
||||
const flags = deriveTerminalFlags(buffered.status);
|
||||
return {
|
||||
spanId: buffered.spanId,
|
||||
parentId: buffered.parentSpanId ?? null,
|
||||
runId: buffered.friendlyId,
|
||||
message: buffered.taskIdentifier ?? "",
|
||||
...flags,
|
||||
level: "TRACE" as const,
|
||||
startTime: buffered.createdAt,
|
||||
durationMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Body for GET /api/v1/runs/:runId/trace when the run is buffered.
|
||||
// Returns the `{ trace: { traceId, rootSpan } }` envelope expected by the
|
||||
// SDK's RetrieveRunTraceResponseBody schema.
|
||||
export function buildSyntheticTraceBody(buffered: SyntheticRun) {
|
||||
const flags = deriveTerminalFlags(buffered.status);
|
||||
return {
|
||||
trace: {
|
||||
traceId: buffered.traceId ?? "",
|
||||
rootSpan: {
|
||||
id: buffered.spanId ?? "",
|
||||
runId: buffered.friendlyId,
|
||||
data: {
|
||||
message: buffered.taskIdentifier ?? "",
|
||||
taskSlug: buffered.taskIdentifier ?? undefined,
|
||||
events: [] as unknown[],
|
||||
startTime: buffered.createdAt,
|
||||
duration: 0,
|
||||
...flags,
|
||||
level: "TRACE" as const,
|
||||
queueName: buffered.queue ?? undefined,
|
||||
machinePreset: buffered.machinePreset ?? undefined,
|
||||
},
|
||||
children: [] as unknown[],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import type { MollifierBuffer } from "@trigger.dev/redis-worker";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import { z } from "zod";
|
||||
import { prisma } from "~/db.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "./mollifierBuffer.server";
|
||||
// Use the webapp-side wrapper (not `deserialiseSnapshot` from
|
||||
// @trigger.dev/redis-worker directly) so this file shares a single
|
||||
// deserialisation path with readFallback.server.ts. The two are
|
||||
// behaviourally identical today (both wrap `JSON.parse`), but pinning
|
||||
// the shared helper keeps the two read-side modules from drifting if
|
||||
// snapshot encoding ever changes.
|
||||
import { deserialiseMollifierSnapshot } from "./mollifierSnapshot.server";
|
||||
|
||||
// Validated subset of a mollifier snapshot — just the fields needed to
|
||||
// rebuild a canonical run-detail URL for a buffered run. Anything else
|
||||
// in the payload is ignored. `safeParse` against this schema replaces
|
||||
// the ad-hoc `as Record<string, unknown>` + `typeof === "string"` checks
|
||||
// that the redirect path used to do by hand; missing or wrong-typed
|
||||
// fields collapse into a single `parsed.success === false` branch.
|
||||
const BufferedSnapshotSchema = z.object({
|
||||
spanId: z.string().optional(),
|
||||
environment: z.object({
|
||||
slug: z.string(),
|
||||
project: z.object({ slug: z.string() }),
|
||||
organization: z.object({ slug: z.string() }),
|
||||
}),
|
||||
});
|
||||
|
||||
export type BufferedRunRedirectInfo = {
|
||||
organizationSlug: string;
|
||||
projectSlug: string;
|
||||
environmentSlug: string;
|
||||
spanId: string | undefined;
|
||||
};
|
||||
|
||||
export type FindBufferedRunRedirectInfoDeps = {
|
||||
getBuffer?: () => MollifierBuffer | null;
|
||||
prismaClient?: PrismaClientOrTransaction;
|
||||
};
|
||||
|
||||
// Resolve the org/project/env slugs needed to build the canonical run-detail
|
||||
// URL for a buffered run. Used by the short-URL redirect routes
|
||||
// (`runs.$runParam`, `@.runs.$runParam`, `projects.v3.$projectRef.runs.$runParam`)
|
||||
// so a customer clicking the trigger-API-returned run link doesn't 404
|
||||
// during the buffered window.
|
||||
//
|
||||
// Authorisation: PG query confirms the requesting user belongs to the
|
||||
// organisation the buffer entry says owns the run. Without this check a
|
||||
// known runId would leak slugs.
|
||||
export async function findBufferedRunRedirectInfo(
|
||||
args: {
|
||||
runFriendlyId: string;
|
||||
userId: string;
|
||||
// Admin impersonation paths bypass org-membership; mirrors the existing
|
||||
// PG-side admin route behaviour (`@.runs.$runParam` doesn't filter by
|
||||
// org membership in the PG query either).
|
||||
skipOrgMembershipCheck?: boolean;
|
||||
},
|
||||
deps: FindBufferedRunRedirectInfoDeps = {}
|
||||
): Promise<BufferedRunRedirectInfo | null> {
|
||||
const buffer = (deps.getBuffer ?? getMollifierBuffer)();
|
||||
const prismaClient = deps.prismaClient ?? prisma;
|
||||
if (!buffer) return null;
|
||||
|
||||
let entry;
|
||||
try {
|
||||
entry = await buffer.getEntry(args.runFriendlyId);
|
||||
} catch (err) {
|
||||
logger.warn("buffered redirect: buffer.getEntry failed", {
|
||||
runFriendlyId: args.runFriendlyId,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
if (!entry) return null;
|
||||
|
||||
if (!args.skipOrgMembershipCheck) {
|
||||
const member = await prismaClient.orgMember.findFirst({
|
||||
where: { userId: args.userId, organizationId: entry.orgId },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!member) return null;
|
||||
}
|
||||
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = deserialiseMollifierSnapshot(entry.payload);
|
||||
} catch (err) {
|
||||
logger.warn("buffered redirect: snapshot deserialise failed", {
|
||||
runFriendlyId: args.runFriendlyId,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = BufferedSnapshotSchema.safeParse(raw);
|
||||
if (!parsed.success) {
|
||||
// Either the snapshot is from a different writer that doesn't carry
|
||||
// environment slugs (in which case we genuinely can't build a URL)
|
||||
// or a buffer-format drift snuck through. Log at debug; the caller
|
||||
// 404s and the user sees the standard not-found page, not a 500.
|
||||
logger.debug("buffered redirect: snapshot shape mismatch", {
|
||||
runFriendlyId: args.runFriendlyId,
|
||||
issues: parsed.error.issues.map((issue) => ({
|
||||
path: issue.path.join("."),
|
||||
code: issue.code,
|
||||
})),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
organizationSlug: parsed.data.environment.organization.slug,
|
||||
projectSlug: parsed.data.environment.project.slug,
|
||||
environmentSlug: parsed.data.environment.slug,
|
||||
spanId: parsed.data.spanId,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import type { SyntheticRun } from "./readFallback.server";
|
||||
|
||||
export type SyntheticReplayTaskRun = TaskRun & {
|
||||
project: { slug: string; organization: { slug: string } };
|
||||
runtimeEnvironment: { slug: string };
|
||||
};
|
||||
|
||||
// Adapt a buffered-run snapshot into the TaskRun-shaped input that
|
||||
// `ReplayTaskRunService.call` expects. ReplayTaskRunService builds the
|
||||
// new run's traceparent as `00-${existingTaskRun.traceId}-${existingTaskRun.spanId}-01`
|
||||
// without guarding for undefined, so a synthetic with missing traceId
|
||||
// or spanId (older snapshots — both fields are documented optional on
|
||||
// `SyntheticRun`) would produce `00-undefined-undefined-01`, an invalid
|
||||
// W3C traceparent that OTel silently drops, severing the replay's trace
|
||||
// link to the original run.
|
||||
//
|
||||
// Returns null when those fields are missing — the caller surfaces this
|
||||
// as "Run not found" so the customer retries once the drainer has
|
||||
// materialised the PG row, where traceId/spanId are guaranteed present.
|
||||
export function buildSyntheticReplayTaskRun(args: {
|
||||
synthetic: SyntheticRun;
|
||||
envRow: {
|
||||
slug: string;
|
||||
project: { slug: string; organization: { slug: string } };
|
||||
};
|
||||
}): SyntheticReplayTaskRun | null {
|
||||
const { synthetic, envRow } = args;
|
||||
if (!synthetic.traceId || !synthetic.spanId) return null;
|
||||
return {
|
||||
// The double `as unknown as TaskRun` cast is load-bearing — a direct
|
||||
// `synthetic as TaskRun` won't compile. `SyntheticRun` carries the
|
||||
// subset of fields that `ReplayTaskRunService.call` actually reads
|
||||
// (the contract is enumerated on the SyntheticRun type comment in
|
||||
// readFallback.server.ts), but its shape is not structurally
|
||||
// assignable to the full Prisma `TaskRun` row: optional vs required
|
||||
// fields diverge, several PG columns (number, batchId variants,
|
||||
// status enum widening) are deliberately absent or narrower on the
|
||||
// synthetic. Routing it through `unknown` is the explicit "we know
|
||||
// this is a subset, we've audited which fields are read" signal,
|
||||
// and the traceId/spanId guard above prevents the only field
|
||||
// ReplayTaskRunService consumes that would corrupt downstream
|
||||
// behaviour (the OTel traceparent) when undefined.
|
||||
...(synthetic as unknown as TaskRun),
|
||||
project: {
|
||||
slug: envRow.project.slug,
|
||||
organization: { slug: envRow.project.organization.slug },
|
||||
},
|
||||
runtimeEnvironment: { slug: envRow.slug },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { SyntheticRun } from "./readFallback.server";
|
||||
|
||||
// Synthesise the run-detail page's `run` header shape (the NavBar +
|
||||
// status badge + Cancel-button gate) from a buffered run snapshot. The
|
||||
// shape matches `RunPresenter.getRun`'s `runData` — keep this in sync
|
||||
// when fields are added there.
|
||||
//
|
||||
// CANCELED and FAILED state is reflected back from
|
||||
// `SyntheticRun.cancelledAt` / `status` so terminal buffered runs show
|
||||
// the correct status in the NavBar + isFinished:true (which collapses
|
||||
// the Cancel button on the page header) before the drainer materialises
|
||||
// the PG row. This mirrors what `buildSyntheticSpanRun` does for the
|
||||
// right-side details panel — the SyntheticRun.cancelledAt contract
|
||||
// comment in readFallback.server.ts names this exact UI surface.
|
||||
//
|
||||
// FAILED status maps to `SYSTEM_FAILURE` to match the drainer's
|
||||
// non-retryable terminal path, which is what `buildSyntheticSpanRun`
|
||||
// uses too. Symmetric across the header + span-detail panel so an
|
||||
// admin doesn't see "Pending" + "FAILED" simultaneously on the same
|
||||
// run.
|
||||
export function buildSyntheticRunHeader(args: {
|
||||
run: SyntheticRun;
|
||||
environment: {
|
||||
id: string;
|
||||
organizationId: string;
|
||||
type: "PRODUCTION" | "DEVELOPMENT" | "STAGING" | "PREVIEW";
|
||||
slug: string;
|
||||
};
|
||||
}) {
|
||||
const { run, environment } = args;
|
||||
const isCancelled = run.status === "CANCELED";
|
||||
const isFailed = run.status === "FAILED";
|
||||
|
||||
return {
|
||||
// `id` mirrors RunPresenter.getRun's runData (the PG path), which
|
||||
// is the internal cuid — not the friendlyId. SyntheticRun.id is
|
||||
// already the cuid (RunId.fromFriendlyId(entry.runId) in
|
||||
// readFallback.server.ts) so the admin debug tooltip on the run
|
||||
// detail page shows the same format for buffered + materialised
|
||||
// runs.
|
||||
id: run.id,
|
||||
number: 1,
|
||||
friendlyId: run.friendlyId,
|
||||
traceId: run.traceId ?? "",
|
||||
spanId: run.spanId ?? "",
|
||||
status: isCancelled
|
||||
? ("CANCELED" as const)
|
||||
: isFailed
|
||||
? ("SYSTEM_FAILURE" as const)
|
||||
: ("PENDING" as const),
|
||||
isFinished: isCancelled || isFailed,
|
||||
startedAt: null,
|
||||
// Symmetric with `buildSyntheticSpanRun` and the
|
||||
// `ApiRetrieveRunPresenter` synth path. The run-detail route
|
||||
// derives `isCompleted` from `completedAt !== null` and gates SSE
|
||||
// live-reloading on it (`route.tsx:459`, `:551`); leaving
|
||||
// `completedAt` null for FAILED would keep a terminal buffered run
|
||||
// live-reloading forever. PG-resident SYSTEM_FAILURE rows always
|
||||
// have completedAt set, so fall back to createdAt (the buffer
|
||||
// entry has no separate failedAt — closest proxy for when the
|
||||
// terminal state landed).
|
||||
completedAt: run.cancelledAt ?? (isFailed ? run.createdAt : null),
|
||||
logsDeletedAt: null,
|
||||
rootTaskRun: null,
|
||||
parentTaskRun: null,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
organizationId: environment.organizationId,
|
||||
type: environment.type,
|
||||
slug: environment.slug,
|
||||
userId: undefined,
|
||||
userName: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import { prettyPrintPacket, RunAnnotations } from "@trigger.dev/core/v3";
|
||||
import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
extractIdempotencyKeyScope,
|
||||
getUserProvidedIdempotencyKey,
|
||||
} from "@trigger.dev/core/v3/serverOnly";
|
||||
import { MachinePresetName } from "@trigger.dev/core/v3/schemas";
|
||||
import type { SpanRun } from "~/presenters/v3/SpanPresenter.server";
|
||||
import type { SyntheticRun } from "./readFallback.server";
|
||||
|
||||
// `SyntheticRun.machinePreset` is sourced from the snapshot payload as
|
||||
// a plain string, but `SpanRun.machinePreset` is the narrowed enum.
|
||||
// Validate against the canonical enum so an unknown / stale preset
|
||||
// string collapses to undefined rather than fighting the type checker.
|
||||
function narrowMachinePreset(value: string | undefined): SpanRun["machinePreset"] {
|
||||
if (value === undefined) return undefined;
|
||||
const parsed = MachinePresetName.safeParse(value);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
|
||||
// Synthesise a SpanRun-shaped object from a buffered run so the run-detail
|
||||
// page's right-side details panel renders identically to a PG-resident
|
||||
// run. The shape matches `SpanPresenter.getRun`'s return value;
|
||||
// buffered-irrelevant fields (output, attempts, schedule, session,
|
||||
// region, batch) are filled with sensible defaults, while terminal state
|
||||
// (CANCELED / FAILED) is reflected into `status`, `isFinished`, `isError`
|
||||
// and `error` so a finished buffered run does not render as PENDING.
|
||||
//
|
||||
// Pretty-printing for payload and metadata mirrors SpanPresenter so the
|
||||
// UI receives data in the same shape. Buffered runs cannot use the
|
||||
// `application/store` packet path (no R2 object yet) so we treat raw
|
||||
// snapshot fields as inline packets.
|
||||
export async function buildSyntheticSpanRun(args: {
|
||||
run: SyntheticRun;
|
||||
environment: {
|
||||
id: string;
|
||||
slug: string;
|
||||
type: "PRODUCTION" | "DEVELOPMENT" | "STAGING" | "PREVIEW";
|
||||
};
|
||||
}): Promise<SpanRun> {
|
||||
const { run, environment } = args;
|
||||
|
||||
const payload =
|
||||
typeof run.payload !== "undefined" && run.payload !== null
|
||||
? await prettyPrintPacket(run.payload, run.payloadType ?? undefined)
|
||||
: undefined;
|
||||
|
||||
// Nullish check, not truthy — matches the payload branch above so an
|
||||
// intentionally-empty packet (e.g. metadata: "") still gets handed to
|
||||
// `prettyPrintPacket` and renders consistently. A truthy check would
|
||||
// drop the empty-string case and the two paths would diverge.
|
||||
const metadata =
|
||||
typeof run.metadata !== "undefined" && run.metadata !== null
|
||||
? await prettyPrintPacket(run.metadata, run.metadataType, {
|
||||
filteredKeys: ["$$streams", "$$streamsVersion", "$$streamsBaseUrl"],
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const idempotencyShape = {
|
||||
idempotencyKey: run.idempotencyKey ?? null,
|
||||
idempotencyKeyExpiresAt: null,
|
||||
idempotencyKeyOptions: run.idempotencyKeyOptions ?? null,
|
||||
};
|
||||
|
||||
const idempotencyKey = getUserProvidedIdempotencyKey(idempotencyShape);
|
||||
const idempotencyKeyScope = extractIdempotencyKeyScope(idempotencyShape);
|
||||
const idempotencyKeyStatus: SpanRun["idempotencyKeyStatus"] = idempotencyKey
|
||||
? "active"
|
||||
: idempotencyKeyScope
|
||||
? "inactive"
|
||||
: undefined;
|
||||
|
||||
const taskKind = RunAnnotations.safeParse(run.annotations).data?.taskKind;
|
||||
const isAgentRun = taskKind === "AGENT";
|
||||
const isScheduled = taskKind === "SCHEDULED";
|
||||
|
||||
const queueName = run.queue ?? "task/";
|
||||
const isCancelled = run.status === "CANCELED";
|
||||
const isFailed = run.status === "FAILED";
|
||||
|
||||
// The run-detail panel derives terminal/error state from `status`,
|
||||
// `isFinished` and `isError` (SpanPresenter.getRun -> isFinalRunStatus /
|
||||
// isFailedRunStatus). Buffered FAILED runs surface as SYSTEM_FAILURE to
|
||||
// match ApiRetrieveRunPresenter.bufferedStatusToTaskRunStatus; both
|
||||
// CANCELED and SYSTEM_FAILURE are final run statuses, and SYSTEM_FAILURE
|
||||
// is also a failed status.
|
||||
const status: SpanRun["status"] = isCancelled
|
||||
? "CANCELED"
|
||||
: isFailed
|
||||
? "SYSTEM_FAILURE"
|
||||
: "PENDING";
|
||||
|
||||
// Mirror ApiRetrieveRunPresenter's STRING_ERROR synthesis so the panel
|
||||
// shows why a buffered run failed instead of an empty error block.
|
||||
const error: SpanRun["error"] =
|
||||
isFailed && run.error
|
||||
? { type: "STRING_ERROR", raw: `${run.error.code}: ${run.error.message}` }
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
id: run.id,
|
||||
friendlyId: run.friendlyId,
|
||||
status,
|
||||
statusReason: isCancelled
|
||||
? (run.cancelReason ?? undefined)
|
||||
: isFailed
|
||||
? (run.error?.message ?? undefined)
|
||||
: undefined,
|
||||
createdAt: run.createdAt,
|
||||
startedAt: null,
|
||||
executedAt: null,
|
||||
updatedAt: run.cancelledAt ?? run.createdAt,
|
||||
delayUntil: run.delayUntil ?? null,
|
||||
expiredAt: null,
|
||||
// Symmetric with `ApiRetrieveRunPresenter` — FAILED buffered runs
|
||||
// must surface a non-null `completedAt` so the run-detail panel
|
||||
// (and any caller checking `isFinished && completedAt`) doesn't
|
||||
// render a finished run with no completion timestamp. PG-resident
|
||||
// SYSTEM_FAILURE rows always have completedAt set; the buffer
|
||||
// entry has no separate failedAt, so we fall back to createdAt
|
||||
// as the best proxy for when the terminal state landed.
|
||||
completedAt: run.cancelledAt ?? (isFailed ? run.createdAt : null),
|
||||
logsDeletedAt: null,
|
||||
ttl: run.ttl ?? null,
|
||||
taskIdentifier: run.taskIdentifier ?? "",
|
||||
version: undefined,
|
||||
sdkVersion: undefined,
|
||||
runtime: undefined,
|
||||
runtimeVersion: undefined,
|
||||
isTest: run.isTest,
|
||||
replayedFromTaskRunFriendlyId: run.replayedFromTaskRunFriendlyId ?? null,
|
||||
environmentId: environment.id,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt: null,
|
||||
idempotencyKeyScope,
|
||||
idempotencyKeyStatus,
|
||||
debounce: null,
|
||||
schedule: undefined,
|
||||
queue: {
|
||||
name: queueName,
|
||||
isCustomQueue: !queueName.startsWith("task/"),
|
||||
concurrencyKey: run.concurrencyKey ?? null,
|
||||
},
|
||||
tags: run.runTags,
|
||||
baseCostInCents: 0,
|
||||
costInCents: 0,
|
||||
totalCostInCents: 0,
|
||||
usageDurationMs: 0,
|
||||
isFinished: isCancelled || isFailed,
|
||||
isRunning: false,
|
||||
isError: isFailed,
|
||||
isAgentRun,
|
||||
isScheduled,
|
||||
payload,
|
||||
payloadType: run.payloadType ?? "application/json",
|
||||
output: undefined,
|
||||
outputType: "application/json",
|
||||
error,
|
||||
// The snapshot only carries the root/parent friendly IDs, not the
|
||||
// spanId or taskIdentifier that SpanPresenter sources from the joined
|
||||
// PG rows. Emitting them with empty-string stubs renders a blank task
|
||||
// name and a misleading `?span=` jump target, so we omit the
|
||||
// relationships until the drainer materialises the row (a transient
|
||||
// window). Top-level buffered runs have no relationships regardless.
|
||||
relationships: {
|
||||
root: undefined,
|
||||
parent: undefined,
|
||||
},
|
||||
context: JSON.stringify(
|
||||
{
|
||||
task: {
|
||||
id: run.taskIdentifier ?? "",
|
||||
},
|
||||
run: {
|
||||
id: run.friendlyId,
|
||||
createdAt: run.createdAt,
|
||||
isTest: run.isTest,
|
||||
},
|
||||
environment: {
|
||||
id: environment.id,
|
||||
slug: environment.slug,
|
||||
type: environment.type,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2
|
||||
),
|
||||
metadata,
|
||||
maxDurationInSeconds: getMaxDuration(run.maxDurationInSeconds),
|
||||
batch: undefined,
|
||||
session: undefined,
|
||||
engine: "V2",
|
||||
region: null,
|
||||
workerQueue: run.workerQueue ?? "",
|
||||
traceId: run.traceId ?? "",
|
||||
spanId: run.spanId ?? "",
|
||||
isCached: false,
|
||||
isBuffered: true,
|
||||
machinePreset: narrowMachinePreset(run.machinePreset),
|
||||
taskEventStore: "taskEvent",
|
||||
externalTraceId: undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { millisecondsToNanoseconds } from "@trigger.dev/core/v3";
|
||||
import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
|
||||
import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
|
||||
import type { SpanSummary } from "~/v3/eventRepository/eventRepository.types";
|
||||
import type { SyntheticRun } from "./readFallback.server";
|
||||
|
||||
// Build a single-span trace for a buffered run so the run-detail page
|
||||
// renders a meaningful timeline before the drainer materialises the
|
||||
// row. Mirrors the shape produced by `RunPresenter` when its trace
|
||||
// store lookup returns no spans, so the dashboard consumer treats the
|
||||
// buffered run identically to a freshly enqueued PG run that hasn't
|
||||
// emitted any events yet.
|
||||
export function buildSyntheticTraceForBufferedRun(run: SyntheticRun) {
|
||||
const spanId = run.spanId ?? "";
|
||||
const isCancelled = run.status === "CANCELED";
|
||||
const isFailed = run.status === "FAILED";
|
||||
const span: SpanSummary = {
|
||||
id: spanId,
|
||||
parentId: run.parentSpanId,
|
||||
runId: run.friendlyId,
|
||||
data: {
|
||||
message: run.taskIdentifier ?? "Task",
|
||||
style: { icon: "task", variant: "primary" },
|
||||
events: [],
|
||||
startTime: run.createdAt,
|
||||
duration: 0,
|
||||
isError: isFailed,
|
||||
// CANCELED and FAILED are terminal; only a still-queued buffered run
|
||||
// is partial. A partial failed span would otherwise render as
|
||||
// "executing" forever in the timeline.
|
||||
isPartial: !isCancelled && !isFailed,
|
||||
isCancelled,
|
||||
isDebug: false,
|
||||
level: "TRACE",
|
||||
},
|
||||
};
|
||||
|
||||
const tree = createTreeFromFlatItems([span], spanId);
|
||||
const treeRootStartTimeMs = tree?.data.startTime.getTime() ?? 0;
|
||||
const totalDuration = Math.max(tree?.data.duration ?? 0, millisecondsToNanoseconds(1));
|
||||
|
||||
const events = tree
|
||||
? flattenTree(tree).map((n) => {
|
||||
const offset = millisecondsToNanoseconds(n.data.startTime.getTime() - treeRootStartTimeMs);
|
||||
// Mirror RunPresenter: raw span events stay server-side, only
|
||||
// timelineEvents ship to the client.
|
||||
const { events: spanEvents, ...data } = n.data;
|
||||
return {
|
||||
...n,
|
||||
data: {
|
||||
...data,
|
||||
timelineEvents: createTimelineSpanEventsFromSpanEvents(
|
||||
spanEvents,
|
||||
false,
|
||||
treeRootStartTimeMs
|
||||
),
|
||||
duration: n.data.isPartial ? null : n.data.duration,
|
||||
offset,
|
||||
isRoot: n.id === spanId,
|
||||
// Synthetic traces represent buffered/queued/canceled runs that
|
||||
// haven't executed yet — they can't be agent runs. Keeping this
|
||||
// field present (instead of omitting it) keeps the trace shape
|
||||
// structurally identical to `RunPresenter`'s, which downstream
|
||||
// consumers like the agent-icon check in runs/$runParam/route
|
||||
// require to typecheck against the JsonifyObject union.
|
||||
isAgentRun: false,
|
||||
},
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
return {
|
||||
// Matches RunPresenter's derivation: failed root span -> "failed",
|
||||
// otherwise a terminal (non-partial) span -> "completed", else
|
||||
// "executing". CANCELED is terminal-but-not-error, so "completed".
|
||||
rootSpanStatus: (isFailed ? "failed" : isCancelled ? "completed" : "executing") as
|
||||
| "executing"
|
||||
| "completed"
|
||||
| "failed",
|
||||
events,
|
||||
duration: totalDuration,
|
||||
rootStartedAt: tree?.data.startTime,
|
||||
startedAt: null,
|
||||
queuedDuration: undefined,
|
||||
overridesBySpanId: undefined,
|
||||
linkedRunIdBySpanId: {} as Record<string, string>,
|
||||
isTruncated: false,
|
||||
missingAnchor: false,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user