chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,35 @@
|
||||
import { Ratelimit } from "@upstash/ratelimit";
|
||||
import type { GlobalRateLimiter } from "@trigger.dev/redis-worker";
|
||||
import { RateLimiter } from "~/services/rateLimiter.server";
|
||||
|
||||
/**
|
||||
* Creates a global rate limiter for the batch queue that limits
|
||||
* the maximum number of items processed per second across all consumers.
|
||||
*
|
||||
* Uses a token bucket algorithm where:
|
||||
* - `itemsPerSecond` tokens are available per second
|
||||
* - The bucket can hold up to `itemsPerSecond` tokens (burst capacity)
|
||||
*
|
||||
* @param itemsPerSecond - Maximum items to process per second
|
||||
* @returns A GlobalRateLimiter compatible with FairQueue
|
||||
*/
|
||||
export function createBatchGlobalRateLimiter(itemsPerSecond: number): GlobalRateLimiter {
|
||||
const limiter = new RateLimiter({
|
||||
keyPrefix: "batch-queue-global",
|
||||
// Token bucket: refills `itemsPerSecond` tokens every second
|
||||
// Bucket capacity is also `itemsPerSecond` (allows burst up to limit)
|
||||
limiter: Ratelimit.tokenBucket(itemsPerSecond, "1 s", itemsPerSecond),
|
||||
logSuccess: false,
|
||||
logFailure: true,
|
||||
});
|
||||
|
||||
return {
|
||||
async limit() {
|
||||
const result = await limiter.limit("global");
|
||||
return {
|
||||
allowed: result.success,
|
||||
resetAt: result.reset,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { z } from "zod";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
createLimiterFromConfig,
|
||||
RateLimiterConfig,
|
||||
} from "~/services/authorizationRateLimitMiddleware.server";
|
||||
import type { Duration } from "~/services/rateLimiter.server";
|
||||
import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
const BatchLimitsConfig = z.object({
|
||||
processingConcurrency: z.number().int().default(env.BATCH_CONCURRENCY_LIMIT_DEFAULT),
|
||||
});
|
||||
|
||||
/**
|
||||
* Batch limits configuration for a plan type
|
||||
*/
|
||||
export type BatchLimitsConfig = z.infer<typeof BatchLimitsConfig>;
|
||||
|
||||
const batchLimitsRedisClient = singleton("batchLimitsRedisClient", createBatchLimitsRedisClient);
|
||||
|
||||
function createBatchLimitsRedisClient() {
|
||||
const redisClient = createRedisRateLimitClient({
|
||||
port: env.RATE_LIMIT_REDIS_PORT,
|
||||
host: env.RATE_LIMIT_REDIS_HOST,
|
||||
username: env.RATE_LIMIT_REDIS_USERNAME,
|
||||
password: env.RATE_LIMIT_REDIS_PASSWORD,
|
||||
tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
});
|
||||
|
||||
return redisClient;
|
||||
}
|
||||
|
||||
// Just the org fields this module reads. Compatible with both the full
|
||||
// Prisma `Organization` payload and the slim `AuthenticatedEnvironment`
|
||||
// `["organization"]` shape (when passed `batchRateLimitConfig` /
|
||||
// `batchQueueConcurrencyConfig` as `unknown`).
|
||||
type OrganizationForBatchLimits = {
|
||||
batchRateLimitConfig?: unknown;
|
||||
batchQueueConcurrencyConfig?: unknown;
|
||||
};
|
||||
|
||||
function createOrganizationRateLimiter(organization: OrganizationForBatchLimits): RateLimiter {
|
||||
const limiterConfig = resolveBatchRateLimitConfig(organization.batchRateLimitConfig);
|
||||
|
||||
const limiter = createLimiterFromConfig(limiterConfig);
|
||||
|
||||
return new RateLimiter({
|
||||
redisClient: batchLimitsRedisClient,
|
||||
keyPrefix: "ratelimit:batch",
|
||||
limiter,
|
||||
logSuccess: false,
|
||||
logFailure: true,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveBatchRateLimitConfig(batchRateLimitConfig?: unknown): RateLimiterConfig {
|
||||
const defaultRateLimiterConfig: RateLimiterConfig = {
|
||||
type: "tokenBucket",
|
||||
refillRate: env.BATCH_RATE_LIMIT_REFILL_RATE,
|
||||
interval: env.BATCH_RATE_LIMIT_REFILL_INTERVAL as Duration,
|
||||
maxTokens: env.BATCH_RATE_LIMIT_MAX,
|
||||
};
|
||||
|
||||
if (!batchRateLimitConfig) {
|
||||
return defaultRateLimiterConfig;
|
||||
}
|
||||
|
||||
const parsedBatchRateLimitConfig = RateLimiterConfig.safeParse(batchRateLimitConfig);
|
||||
|
||||
if (!parsedBatchRateLimitConfig.success) {
|
||||
return defaultRateLimiterConfig;
|
||||
}
|
||||
|
||||
return parsedBatchRateLimitConfig.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the rate limiter and limits for an organization.
|
||||
* Internally looks up the plan type, but doesn't expose it to callers.
|
||||
*/
|
||||
export async function getBatchLimits(
|
||||
organization: OrganizationForBatchLimits
|
||||
): Promise<{ rateLimiter: RateLimiter; config: BatchLimitsConfig }> {
|
||||
const rateLimiter = createOrganizationRateLimiter(organization);
|
||||
const config = resolveBatchLimitsConfig(organization.batchQueueConcurrencyConfig);
|
||||
return { rateLimiter, config };
|
||||
}
|
||||
|
||||
function resolveBatchLimitsConfig(batchLimitsConfig?: unknown): BatchLimitsConfig {
|
||||
const defaultLimitsConfig: BatchLimitsConfig = {
|
||||
processingConcurrency: env.BATCH_CONCURRENCY_LIMIT_DEFAULT,
|
||||
};
|
||||
|
||||
if (!batchLimitsConfig) {
|
||||
return defaultLimitsConfig;
|
||||
}
|
||||
|
||||
const parsedBatchLimitsConfig = BatchLimitsConfig.safeParse(batchLimitsConfig);
|
||||
|
||||
if (!parsedBatchLimitsConfig.success) {
|
||||
return defaultLimitsConfig;
|
||||
}
|
||||
|
||||
return parsedBatchLimitsConfig.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when batch rate limit is exceeded.
|
||||
* Contains information for constructing a proper 429 response.
|
||||
*/
|
||||
export class BatchRateLimitExceededError extends Error {
|
||||
constructor(
|
||||
public readonly limit: number,
|
||||
public readonly remaining: number,
|
||||
public readonly resetAt: Date,
|
||||
public readonly itemCount: number
|
||||
) {
|
||||
super(`Batch rate limit exceeded. Limit resets at ${resetAt.toISOString()}`);
|
||||
this.name = "BatchRateLimitExceededError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import pRetry from "p-retry";
|
||||
import { env } from "~/env.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { hasObjectStoreClient, uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { startActiveSpan } from "~/v3/tracer.server";
|
||||
|
||||
export type BatchPayloadProcessResult = {
|
||||
/** The processed payload - either the original or an R2 path */
|
||||
payload: unknown;
|
||||
/** The payload type - "application/store" if offloaded to R2 */
|
||||
payloadType: string;
|
||||
/** Whether the payload was offloaded to R2 */
|
||||
wasOffloaded: boolean;
|
||||
/** Size of the payload in bytes */
|
||||
size: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* BatchPayloadProcessor handles payload offloading for batch items.
|
||||
*
|
||||
* When a batch item's payload exceeds the configured threshold, it's uploaded
|
||||
* to object storage (R2) and the payload is replaced with the storage path.
|
||||
* This aligns with how single task triggers work via DefaultPayloadProcessor.
|
||||
*
|
||||
* Path format: batch_{batchId}/item_{index}/payload.json
|
||||
*/
|
||||
export class BatchPayloadProcessor {
|
||||
/**
|
||||
* Check if object storage is available for payload offloading.
|
||||
* If not available, large payloads will be stored inline (which may fail for very large payloads).
|
||||
*/
|
||||
isObjectStoreAvailable(): boolean {
|
||||
return hasObjectStoreClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a batch item payload, offloading to R2 if it exceeds the threshold.
|
||||
*
|
||||
* @param payload - The raw payload from the batch item
|
||||
* @param payloadType - The payload type (e.g., "application/json")
|
||||
* @param batchId - The batch ID (internal format)
|
||||
* @param itemIndex - The item index within the batch
|
||||
* @param environment - The authenticated environment for R2 path construction
|
||||
* @returns The processed result with potentially offloaded payload
|
||||
*/
|
||||
async process(
|
||||
payload: unknown,
|
||||
payloadType: string,
|
||||
batchId: string,
|
||||
itemIndex: number,
|
||||
environment: AuthenticatedEnvironment
|
||||
): Promise<BatchPayloadProcessResult> {
|
||||
return startActiveSpan("BatchPayloadProcessor.process()", async (span) => {
|
||||
span.setAttribute("batchId", batchId);
|
||||
span.setAttribute("itemIndex", itemIndex);
|
||||
span.setAttribute("payloadType", payloadType);
|
||||
|
||||
// Create the packet for size checking
|
||||
const packet = this.#createPayloadPacket(payload, payloadType);
|
||||
|
||||
if (!packet.data) {
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const threshold = env.BATCH_PAYLOAD_OFFLOAD_THRESHOLD ?? env.TASK_PAYLOAD_OFFLOAD_THRESHOLD;
|
||||
const { needsOffloading, size } = packetRequiresOffloading(packet, threshold);
|
||||
|
||||
span.setAttribute("payloadSize", size);
|
||||
span.setAttribute("needsOffloading", needsOffloading);
|
||||
span.setAttribute("threshold", threshold);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if object store is available
|
||||
if (!this.isObjectStoreAvailable()) {
|
||||
logger.warn("Payload exceeds threshold but object store is not available", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
size,
|
||||
threshold,
|
||||
});
|
||||
|
||||
// Return without offloading - the payload will be stored inline
|
||||
// This may fail downstream for very large payloads
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size,
|
||||
};
|
||||
}
|
||||
|
||||
// Upload to object store, retrying on transient network errors
|
||||
const { data: packetData, dataType: packetDataType } = packet;
|
||||
const filename = `batch_${batchId}/item_${itemIndex}/payload.json`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
pRetry(
|
||||
() =>
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packetData,
|
||||
packetDataType,
|
||||
environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
),
|
||||
{
|
||||
retries: 3,
|
||||
minTimeout: 500,
|
||||
maxTimeout: 2000,
|
||||
factor: 2,
|
||||
onFailedAttempt: (error) => {
|
||||
logger.warn("Batch item payload upload to object store failed, retrying", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
attempt: error.attemptNumber,
|
||||
retriesLeft: error.retriesLeft,
|
||||
error: error.message,
|
||||
});
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
logger.error("Failed to upload batch item payload to object store after retries", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
error: uploadError.message,
|
||||
});
|
||||
|
||||
throw new Error(`Failed to upload large payload to object store: ${uploadError.message}`);
|
||||
}
|
||||
|
||||
logger.debug("Batch item payload offloaded to object store", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
filename: uploadedFilename,
|
||||
size,
|
||||
});
|
||||
|
||||
span.setAttribute("wasOffloaded", true);
|
||||
span.setAttribute("offloadPath", uploadedFilename);
|
||||
|
||||
return {
|
||||
payload: uploadedFilename!,
|
||||
payloadType: "application/store",
|
||||
wasOffloaded: true,
|
||||
size,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOPacket from payload for size checking.
|
||||
*/
|
||||
#createPayloadPacket(payload: unknown, payloadType: string): IOPacket {
|
||||
if (payloadType === "application/json") {
|
||||
// Payload from SDK is already serialized as a string - use directly
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: "application/json" };
|
||||
}
|
||||
// Non-string payloads (e.g., direct API calls with objects) need serialization
|
||||
return { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
}
|
||||
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: payloadType };
|
||||
}
|
||||
|
||||
// For other types, try to stringify
|
||||
try {
|
||||
return { data: JSON.stringify(payload), dataType: payloadType };
|
||||
} catch {
|
||||
return { dataType: payloadType };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { hashBucket } from "~/utils/computeBucket";
|
||||
|
||||
/** Subset of the global flags snapshot this resolver reads. */
|
||||
export type ComputeMigrationFlags = {
|
||||
computeMigrationEnabled?: boolean;
|
||||
computeMigrationFreePercentage?: number;
|
||||
computeMigrationPaidPercentage?: number;
|
||||
};
|
||||
|
||||
type MigrationDecisionInput = {
|
||||
planType: string | undefined;
|
||||
orgId: string;
|
||||
orgFeatureFlags: Record<string, unknown> | null | undefined;
|
||||
flags: ComputeMigrationFlags | undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether this org should run on the compute backing. Shared by the trigger-time
|
||||
* transform and the deploy-time template decision so a migrated org always gets a
|
||||
* compute template. Precedence: per-org override (both directions) wins; otherwise
|
||||
* global enable + the plan's percentage bucket. Enterprise and unknown plans are
|
||||
* never enrolled by percentage (override only). The sole opt-out is the per-org
|
||||
* `computeMigrationEnabled: false`.
|
||||
*/
|
||||
export function isOrgMigrated({
|
||||
planType,
|
||||
orgId,
|
||||
orgFeatureFlags,
|
||||
flags,
|
||||
}: MigrationDecisionInput): boolean {
|
||||
const override = orgFeatureFlags?.["computeMigrationEnabled"];
|
||||
if (override === false) return false;
|
||||
if (override === true) return true;
|
||||
|
||||
if (!(flags?.computeMigrationEnabled ?? false)) return false;
|
||||
|
||||
const pct =
|
||||
planType === "free"
|
||||
? (flags?.computeMigrationFreePercentage ?? 0)
|
||||
: planType === "paid"
|
||||
? (flags?.computeMigrationPaidPercentage ?? 0)
|
||||
: 0; // enterprise / undefined
|
||||
|
||||
return hashBucket(orgId) < pct;
|
||||
}
|
||||
|
||||
type ResolveInput = MigrationDecisionInput & {
|
||||
baseWorkerQueue: string | undefined;
|
||||
baseEnableFastPath: boolean;
|
||||
region: string | undefined; // geo of the base queue (same whether migrated or not)
|
||||
backing: { workerQueue: string; enableFastPath: boolean } | undefined;
|
||||
envType: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Produce the target descriptor `{ workerQueue, region, enableFastPath }` for a
|
||||
* run. When the org is migrated and the region has a compute backing, the queue
|
||||
* and fast-path setting come from the MICROVM backing group; `region` is the geo
|
||||
* either way. Same-geo swap (us-east-1 -> us-east-1-next): any explicit placement
|
||||
* is a geography preference, honored by staying in-region. Applied after region
|
||||
* resolution, mirroring the scheduled-split.
|
||||
*/
|
||||
export function resolveComputeMigration({
|
||||
baseWorkerQueue,
|
||||
baseEnableFastPath,
|
||||
region,
|
||||
backing,
|
||||
envType,
|
||||
...decision
|
||||
}: ResolveInput): {
|
||||
workerQueue: string | undefined;
|
||||
region: string | undefined;
|
||||
enableFastPath: boolean;
|
||||
} {
|
||||
const passthrough = { workerQueue: baseWorkerQueue, region, enableFastPath: baseEnableFastPath };
|
||||
if (baseWorkerQueue === undefined) return passthrough;
|
||||
if (envType === "DEVELOPMENT") return passthrough;
|
||||
if (!isOrgMigrated(decision)) return passthrough;
|
||||
if (!backing) return passthrough;
|
||||
return { workerQueue: backing.workerQueue, region, enableFastPath: backing.enableFastPath };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
baseWorkerQueue,
|
||||
matchesDisabledWorkerQueue,
|
||||
parseDisabledWorkerQueues,
|
||||
} from "./workerQueueSplit.server";
|
||||
|
||||
const meter = getMeter("run-engine-dequeue-gate");
|
||||
|
||||
const blockedDequeueCounter = meter.createCounter("run_engine.dequeue.blocked", {
|
||||
description:
|
||||
"Count of worker dequeue requests refused because the worker queue is gated off via RUN_ENGINE_DEQUEUE_DISABLED_WORKER_QUEUES",
|
||||
});
|
||||
|
||||
const disabledWorkerQueues = parseDisabledWorkerQueues(
|
||||
env.RUN_ENGINE_DEQUEUE_DISABLED_WORKER_QUEUES
|
||||
);
|
||||
|
||||
export function isWorkerQueueDequeueDisabled(workerQueue: string): boolean {
|
||||
return matchesDisabledWorkerQueue(workerQueue, disabledWorkerQueues);
|
||||
}
|
||||
|
||||
export function recordBlockedDequeue(workerQueue: string): void {
|
||||
blockedDequeueCounter.add(1, {
|
||||
worker_queue: workerQueue,
|
||||
region: baseWorkerQueue(workerQueue),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
import type { RunEngine } from "~/v3/runEngine.server";
|
||||
import { shouldIdempotencyKeyBeCleared } from "~/v3/taskStatus";
|
||||
import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
|
||||
import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server";
|
||||
import { claimOrAwait } from "~/v3/mollifier/idempotencyClaim.server";
|
||||
import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server";
|
||||
import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveIdempotencyDedupClient } from "./idempotencyResidency.server";
|
||||
import type { TraceEventConcern, TriggerTaskRequest } from "../types";
|
||||
|
||||
// In-memory per-org mollifier-enabled check, shared with `evaluateGate`
|
||||
// (same `Organization.featureFlags` JSON, no DB read). Used to gate the
|
||||
// pre-gate claim's Redis round-trip so non-mollifier orgs don't pay it
|
||||
// during staged rollout — see the comment above the claim block in
|
||||
// handleTriggerRequest.
|
||||
const resolveOrgMollifierFlag = makeResolveMollifierFlag();
|
||||
|
||||
// Claim ownership context returned to the caller when the
|
||||
// IdempotencyKeyConcern won a pre-gate claim. Caller MUST publish the
|
||||
// winning runId on pipeline success (`publishClaim`) or release the
|
||||
// claim on failure (`releaseClaim`).
|
||||
export type ClaimedIdempotency = {
|
||||
envId: string;
|
||||
taskIdentifier: string;
|
||||
idempotencyKey: string;
|
||||
// Ownership token from `claimOrAwait`. The caller's trigger pipeline
|
||||
// MUST thread this into publishClaim/releaseClaim so the buffer's
|
||||
// compare-and-act protects the slot against a stale predecessor.
|
||||
token: string;
|
||||
};
|
||||
|
||||
export type IdempotencyKeyConcernResult =
|
||||
| { isCached: true; run: TaskRun }
|
||||
| {
|
||||
isCached: false;
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
// Set when this trigger holds a pre-gate claim. The caller's
|
||||
// trigger pipeline MUST resolve the claim by either publishing
|
||||
// the runId on success or releasing on failure. Undefined when
|
||||
// the request has no idempotency key, when the buffer is
|
||||
// unavailable, or when the request is a triggerAndWait (claim
|
||||
// path skipped per plan doc).
|
||||
claim?: ClaimedIdempotency;
|
||||
};
|
||||
|
||||
export class IdempotencyKeyConcern {
|
||||
constructor(
|
||||
private readonly prisma: PrismaClientOrTransaction,
|
||||
private readonly engine: RunEngine,
|
||||
private readonly traceEventConcern: TraceEventConcern
|
||||
) {}
|
||||
|
||||
// Buffer-side idempotency dedup. Resolves an idempotency key against the
|
||||
// mollifier buffer when PG missed. Returns a SyntheticRun cast to
|
||||
// TaskRun so the route handler (which only reads run.id / run.friendlyId)
|
||||
// can echo the buffered run's friendlyId as a cached hit. Returns null
|
||||
// for any failure or miss — buffer outages must not 500 the trigger
|
||||
// hot path; we fail open to "no cache hit" and let the request through.
|
||||
private async findBufferedRunWithIdempotency(
|
||||
environmentId: string,
|
||||
organizationId: string,
|
||||
taskIdentifier: string,
|
||||
idempotencyKey: string
|
||||
): Promise<TaskRun | null> {
|
||||
const buffer = getMollifierBuffer();
|
||||
if (!buffer) return null;
|
||||
|
||||
let bufferedRunId: string | null;
|
||||
try {
|
||||
bufferedRunId = await buffer.lookupIdempotency({
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
idempotencyKey,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error("IdempotencyKeyConcern: buffer lookupIdempotency failed", {
|
||||
environmentId,
|
||||
taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
if (!bufferedRunId) return null;
|
||||
|
||||
const synthetic = await findRunByIdWithMollifierFallback({
|
||||
runId: bufferedRunId,
|
||||
environmentId,
|
||||
organizationId,
|
||||
});
|
||||
if (!synthetic) return null;
|
||||
// PG-resident path enforces idempotency-key expiry below
|
||||
// (`existingRun.idempotencyKeyExpiresAt < new Date()` clears the key
|
||||
// and lets a new run go through). The buffer path needs the same
|
||||
// check — without it a customer who passes `idempotencyKeyTTL: "2s"`
|
||||
// gets the cached buffered runId returned indefinitely, because the
|
||||
// buffer entry persists for its own (hours-long) TTL independent of
|
||||
// the customer's key TTL.
|
||||
//
|
||||
// Returning null isn't enough on its own: the trigger pipeline then
|
||||
// proceeds to `mollifyTrigger`, whose `buffer.accept` Lua dedupes by
|
||||
// `(envId, taskIdentifier, idempotencyKey)` via SETNX on the same
|
||||
// `mollifier:idempotency:*` key and would echo the stale runId as
|
||||
// `duplicate_idempotency`. Clear the buffer-side idempotency
|
||||
// binding (both the lookup and any in-flight claim) so the next
|
||||
// accept goes through as a fresh trigger. Mirrors what
|
||||
// `ResetIdempotencyKeyService` does for the explicit
|
||||
// reset-via-API path.
|
||||
if (synthetic.idempotencyKeyExpiresAt && synthetic.idempotencyKeyExpiresAt < new Date()) {
|
||||
const buffer = getMollifierBuffer();
|
||||
if (buffer) {
|
||||
try {
|
||||
await buffer.resetIdempotency({
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
idempotencyKey,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn("IdempotencyKeyConcern: failed to reset expired buffer idempotency", {
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return synthetic as unknown as TaskRun;
|
||||
}
|
||||
|
||||
async handleTriggerRequest(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined
|
||||
): Promise<IdempotencyKeyConcernResult> {
|
||||
const idempotencyKey = request.options?.idempotencyKey ?? request.body.options?.idempotencyKey;
|
||||
const idempotencyKeyExpiresAt =
|
||||
request.options?.idempotencyKeyExpiresAt ??
|
||||
resolveIdempotencyKeyTTL(request.body.options?.idempotencyKeyTTL) ??
|
||||
new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); // 30 days
|
||||
|
||||
if (!idempotencyKey) {
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// Probe and clears must hit the DB where the would-be run will physically live.
|
||||
const dedupClient = await resolveIdempotencyDedupClient(
|
||||
{
|
||||
environmentForMint: {
|
||||
organizationId: request.environment.organizationId,
|
||||
id: request.environment.id,
|
||||
orgFeatureFlags: request.environment.organization?.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: request.body.options?.parentRunId,
|
||||
},
|
||||
{
|
||||
isSplitEnabled,
|
||||
fallbackClient: this.prisma,
|
||||
newClient: runOpsNewPrisma,
|
||||
legacyClient: runOpsLegacyPrisma,
|
||||
resolveMintKind: resolveRunIdMintKind,
|
||||
// `isMigrated` is intentionally omitted: until a child of a swept
|
||||
// legacy-id parent can be born on the new DB, the swept-marker override
|
||||
// would never change the answer, so a child routes by parent id-shape.
|
||||
}
|
||||
);
|
||||
|
||||
const existingRun = idempotencyKey
|
||||
? await runStore.findRun(
|
||||
{
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: request.taskId,
|
||||
},
|
||||
{
|
||||
include: {
|
||||
associatedWaitpoint: true,
|
||||
},
|
||||
},
|
||||
dedupClient
|
||||
)
|
||||
: undefined;
|
||||
|
||||
// Buffer fallback per the mollifier-idempotency design. PG missed —
|
||||
// the same key may belong to a buffered run that hasn't materialised
|
||||
// yet. Skipped when `resumeParentOnCompletion` is set: blocking a
|
||||
// parent on a buffered child via waitpoint requires a PG row that
|
||||
// doesn't exist yet. The follow-up accept's SETNX in mollifyTrigger
|
||||
// still dedupes the trigger itself; the waitpoint just doesn't fire
|
||||
// for this rare race window.
|
||||
if (!existingRun && idempotencyKey && !request.body.options?.resumeParentOnCompletion) {
|
||||
const buffered = await this.findBufferedRunWithIdempotency(
|
||||
request.environment.id,
|
||||
request.environment.organizationId,
|
||||
request.taskId,
|
||||
idempotencyKey
|
||||
);
|
||||
if (buffered) {
|
||||
return { isCached: true, run: buffered };
|
||||
}
|
||||
}
|
||||
|
||||
if (existingRun) {
|
||||
// The idempotency key has expired
|
||||
if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) {
|
||||
logger.debug("[TriggerTaskService][call] Idempotency key has expired", {
|
||||
idempotencyKey: request.options?.idempotencyKey,
|
||||
run: existingRun,
|
||||
});
|
||||
|
||||
// Update the existing run to remove the idempotency key
|
||||
await runStore.clearIdempotencyKey(
|
||||
{ byId: { runId: existingRun.id, idempotencyKey } },
|
||||
dedupClient
|
||||
);
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// If the existing run failed or was expired, we clear the key and do a new run
|
||||
if (shouldIdempotencyKeyBeCleared(existingRun.status)) {
|
||||
logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", {
|
||||
idempotencyKey: request.options?.idempotencyKey,
|
||||
runStatus: existingRun.status,
|
||||
runId: existingRun.id,
|
||||
});
|
||||
|
||||
// Update the existing run to remove the idempotency key
|
||||
await runStore.clearIdempotencyKey(
|
||||
{ byId: { runId: existingRun.id, idempotencyKey } },
|
||||
dedupClient
|
||||
);
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// We have an idempotent run, so we return it
|
||||
const parentRunId = request.body.options?.parentRunId;
|
||||
const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion;
|
||||
|
||||
//We're using `andWait` so we need to block the parent run with a waitpoint
|
||||
if (resumeParentOnCompletion && parentRunId) {
|
||||
// `parentRunId` comes from the request body and isn't re-validated
|
||||
// here, so confirm the parent run is in the caller's environment
|
||||
// before wiring a waitpoint against it.
|
||||
const parentRunInternalId = RunId.fromFriendlyId(parentRunId);
|
||||
const parentRunInCallerEnv = await runStore.findRun(
|
||||
{
|
||||
id: parentRunInternalId,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
},
|
||||
{ select: { id: true } },
|
||||
this.prisma
|
||||
);
|
||||
if (!parentRunInCallerEnv) {
|
||||
throw new ServiceValidationError("Parent run not found in the calling environment", 404);
|
||||
}
|
||||
|
||||
// Get or create waitpoint lazily (existing run may not have one if it was standalone)
|
||||
let associatedWaitpoint = existingRun.associatedWaitpoint;
|
||||
if (!associatedWaitpoint) {
|
||||
associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({
|
||||
runId: existingRun.id,
|
||||
projectId: request.environment.projectId,
|
||||
environmentId: request.environment.id,
|
||||
});
|
||||
}
|
||||
|
||||
await this.traceEventConcern.traceIdempotentRun(
|
||||
request,
|
||||
parentStore,
|
||||
{
|
||||
existingRun,
|
||||
idempotencyKey,
|
||||
incomplete: associatedWaitpoint.status === "PENDING",
|
||||
isError: associatedWaitpoint.outputIsError,
|
||||
},
|
||||
async (event) => {
|
||||
const spanId =
|
||||
request.options?.parentAsLinkType === "replay"
|
||||
? event.spanId
|
||||
: event.traceparent?.spanId
|
||||
? `${event.traceparent.spanId}:${event.spanId}`
|
||||
: event.spanId;
|
||||
|
||||
await this.engine.blockRunWithWaitpoint({
|
||||
runId: parentRunInternalId,
|
||||
waitpoints: associatedWaitpoint!.id,
|
||||
spanIdToComplete: spanId,
|
||||
batch: request.options?.batchId
|
||||
? {
|
||||
id: request.options.batchId,
|
||||
index: request.options.batchIndex ?? 0,
|
||||
}
|
||||
: undefined,
|
||||
projectId: request.environment.projectId,
|
||||
organizationId: request.environment.organizationId,
|
||||
tx: dedupClient,
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return { isCached: true, run: existingRun };
|
||||
}
|
||||
|
||||
// Pre-gate claim — closes the PG+buffer race during gate transition.
|
||||
// All same-key triggers serialise here before evaluateGate decides
|
||||
// PG-pass-through vs mollify. Skipped for triggerAndWait
|
||||
// (resumeParentOnCompletion) — that path bypasses the gate entirely
|
||||
// and its existing PG-side dedup is sufficient.
|
||||
//
|
||||
// Gated on the same per-org mollifier flag the gate uses, and the same
|
||||
// bypass list (debounce + oneTimeUseToken): if the gate would never mollify
|
||||
// the request, there's no buffer to serialise against and PG's unique
|
||||
// constraint already deduplicates concurrent same-key races. Skipping the
|
||||
// claim's Redis SETNX keeps its RTT off the hot path for those requests
|
||||
// during staged rollout. The org-flag check is a pure in-memory read of
|
||||
// `Organization.featureFlags`, no DB query.
|
||||
const claimEligible =
|
||||
!request.body.options?.resumeParentOnCompletion &&
|
||||
!request.body.options?.debounce &&
|
||||
!request.options?.oneTimeUseToken &&
|
||||
(await resolveOrgMollifierFlag({
|
||||
envId: request.environment.id,
|
||||
orgId: request.environment.organizationId,
|
||||
taskId: request.taskId,
|
||||
orgFeatureFlags:
|
||||
(request.environment.organization?.featureFlags as
|
||||
| Record<string, unknown>
|
||||
| null
|
||||
| undefined) ?? null,
|
||||
}));
|
||||
if (claimEligible) {
|
||||
const ttlSeconds = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS,
|
||||
Math.ceil((idempotencyKeyExpiresAt.getTime() - Date.now()) / 1000)
|
||||
)
|
||||
);
|
||||
const outcome = await claimOrAwait({
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
idempotencyKey,
|
||||
ttlSeconds,
|
||||
safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS,
|
||||
pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS,
|
||||
});
|
||||
if (outcome.kind === "resolved") {
|
||||
// Another concurrent trigger committed first. Re-resolve via the
|
||||
// existing checks: writer-side PG findFirst first (defeats
|
||||
// replica lag), then buffer fallback for the buffered case.
|
||||
const writerRun = await runStore.findRun(
|
||||
{
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: request.taskId,
|
||||
},
|
||||
{ include: { associatedWaitpoint: true } },
|
||||
dedupClient
|
||||
);
|
||||
if (writerRun) {
|
||||
return { isCached: true, run: writerRun };
|
||||
}
|
||||
const buffered = await this.findBufferedRunWithIdempotency(
|
||||
request.environment.id,
|
||||
request.environment.organizationId,
|
||||
request.taskId,
|
||||
idempotencyKey
|
||||
);
|
||||
if (buffered) {
|
||||
return { isCached: true, run: buffered };
|
||||
}
|
||||
// Claim resolved to a runId nothing can find — the run was genuinely
|
||||
// lost (claimant errored after publish, or both the PG row and buffer
|
||||
// entry TTL'd out). Terminal, not transient, so falling through to a
|
||||
// fresh trigger is the correct recovery.
|
||||
//
|
||||
// Falling through claimless doesn't duplicate runs: concurrent
|
||||
// fall-throughs converge on one run via the same dedup backstops the
|
||||
// claim layer relies on — PG's unique constraint on the idempotency key
|
||||
// (pass-through path) and `accept`'s SETNX (mollify path). Once the
|
||||
// first commits, later callers find it via the writer-PG / buffer
|
||||
// lookups above despite the stale `resolved:` slot (cleared by its ~30s
|
||||
// TTL). Residual cost is a few deduped trigger attempts, not dup runs.
|
||||
logger.warn("idempotency claim resolved but runId not findable", {
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
claimedRunId: outcome.runId,
|
||||
});
|
||||
}
|
||||
if (outcome.kind === "timed_out") {
|
||||
throw new ServiceValidationError("Idempotency claim resolution timed out", 503);
|
||||
}
|
||||
if (outcome.kind === "claimed") {
|
||||
// Caller MUST publish/release. Signalled via the result's
|
||||
// `claim` field, including the ownership token so the buffer
|
||||
// can compare-and-act on the slot we now own.
|
||||
return {
|
||||
isCached: false,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
claim: {
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
idempotencyKey,
|
||||
token: outcome.token,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
resolveIdempotencyDedupClient,
|
||||
type ResolveIdempotencyClientDeps,
|
||||
} from "./idempotencyResidency.server";
|
||||
|
||||
// Distinct sentinel objects so we can assert WHICH client was selected by reference.
|
||||
const FALLBACK = { __tag: "fallback" } as never;
|
||||
const NEW_CLIENT = { __tag: "new" } as never;
|
||||
const LEGACY_CLIENT = { __tag: "legacy" } as never;
|
||||
|
||||
function makeDeps(over: Partial<ResolveIdempotencyClientDeps>): ResolveIdempotencyClientDeps {
|
||||
return {
|
||||
isSplitEnabled: async () => true,
|
||||
fallbackClient: FALLBACK,
|
||||
newClient: NEW_CLIENT,
|
||||
legacyClient: LEGACY_CLIENT,
|
||||
resolveMintKind: async () => "runOpsId",
|
||||
classify: (id) => {
|
||||
if (id.length === 26 && id[25] === "1") return "NEW";
|
||||
if (id.length === 25) return "LEGACY";
|
||||
throw new Error(`unclassifiable: ${id.length}`);
|
||||
},
|
||||
isMigrated: undefined,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const env = { organizationId: "org_1", id: "env_1", orgFeatureFlags: {} };
|
||||
|
||||
describe("resolveIdempotencyDedupClient", () => {
|
||||
it("returns the fallback client unchanged when split is disabled", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ isSplitEnabled: async () => false })
|
||||
);
|
||||
expect(client).toBe(FALLBACK);
|
||||
});
|
||||
|
||||
it("routes a root run to the NEW client when the env mints run-ops ids", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ resolveMintKind: async () => "runOpsId" })
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a root run to the LEGACY client when the env mints cuid", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ resolveMintKind: async () => "cuid" })
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a child to the NEW client when the run-ops parent is NEW-resident", async () => {
|
||||
const runOpsParent = RunId.toFriendlyId("a".repeat(24) + "01");
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: runOpsParent },
|
||||
makeDeps({ resolveMintKind: async () => "cuid" }) // mint flag must NOT win for a child
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a child to the LEGACY client when the cuid parent is LEGACY-resident", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("b".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ resolveMintKind: async () => "runOpsId" }) // mint flag must NOT win for a child
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a swept (migrated) cuid-parent child to the NEW client", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("c".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ isMigrated: async () => true })
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a non-migrated cuid-parent child to the LEGACY client even when isMigrated is provided", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("d".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ isMigrated: async () => false })
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("falls back to the fallback client when a present parent id is unclassifiable", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" },
|
||||
makeDeps({})
|
||||
);
|
||||
expect(client).toBe(FALLBACK);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { ownerEngine, RunId, type Residency } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
|
||||
type MintKind = "cuid" | "runOpsId";
|
||||
|
||||
export type ResolveIdempotencyClientDeps = {
|
||||
isSplitEnabled: () => Promise<boolean>;
|
||||
fallbackClient: PrismaClientOrTransaction;
|
||||
newClient: PrismaClientOrTransaction;
|
||||
legacyClient: PrismaClientOrTransaction;
|
||||
resolveMintKind: (environment: {
|
||||
organizationId: string;
|
||||
id: string;
|
||||
orgFeatureFlags?: unknown;
|
||||
}) => Promise<MintKind>;
|
||||
classify?: (id: string) => Residency;
|
||||
isMigrated?: (id: string) => Promise<boolean>;
|
||||
};
|
||||
|
||||
export async function resolveIdempotencyDedupClient(
|
||||
args: {
|
||||
environmentForMint: { organizationId: string; id: string; orgFeatureFlags?: unknown };
|
||||
parentRunFriendlyId: string | undefined;
|
||||
},
|
||||
deps: ResolveIdempotencyClientDeps
|
||||
): Promise<PrismaClientOrTransaction> {
|
||||
if (!(await deps.isSplitEnabled())) {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
|
||||
const classify = deps.classify ?? ownerEngine;
|
||||
const clientFor = (residency: Residency): PrismaClientOrTransaction =>
|
||||
residency === "NEW" ? deps.newClient : deps.legacyClient;
|
||||
|
||||
if (args.parentRunFriendlyId) {
|
||||
let parentInternalId: string;
|
||||
try {
|
||||
parentInternalId = RunId.fromFriendlyId(args.parentRunFriendlyId);
|
||||
} catch {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
let residency: Residency;
|
||||
try {
|
||||
residency = classify(parentInternalId);
|
||||
} catch {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
if (residency === "LEGACY" && deps.isMigrated && (await deps.isMigrated(parentInternalId))) {
|
||||
return deps.newClient;
|
||||
}
|
||||
return clientFor(residency);
|
||||
}
|
||||
|
||||
const kind = await deps.resolveMintKind(args.environmentForMint);
|
||||
return clientFor(kind === "runOpsId" ? "NEW" : "LEGACY");
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { IOPacket } from "@trigger.dev/core/v3";
|
||||
import { packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import type { PayloadProcessor, TriggerTaskRequest } from "../types";
|
||||
import { env } from "~/env.server";
|
||||
import { startActiveSpan } from "~/v3/tracer.server";
|
||||
import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
export class DefaultPayloadProcessor implements PayloadProcessor {
|
||||
async process(request: TriggerTaskRequest): Promise<IOPacket> {
|
||||
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
||||
const payload = request.body.payload;
|
||||
const payloadType = request.body.options?.payloadType ?? "application/json";
|
||||
|
||||
const packet = this.#createPayloadPacket(payload, payloadType);
|
||||
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading, size } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
span.setAttribute("needsOffloading", needsOffloading);
|
||||
// When the caller already offloaded the payload (payloadType "application/store"), the
|
||||
// packet here is just the small object-store reference, so `size` measures the reference,
|
||||
// not the payload. Prefer the caller-reported pre-offload size when it's provided so the
|
||||
// span reflects the real payload size. For inline payloads the two agree.
|
||||
span.setAttribute("size", request.body.options?.payloadSize ?? size);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${request.friendlyId}/payload.json`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
request.environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
throw new ServiceValidationError("Failed to upload large payload to object store", 500); // This is retryable
|
||||
}
|
||||
|
||||
return {
|
||||
data: uploadedFilename!,
|
||||
dataType: "application/store",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
#createPayloadPacket(payload: any, payloadType: string): IOPacket {
|
||||
if (payloadType === "application/json") {
|
||||
return { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
}
|
||||
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: payloadType };
|
||||
}
|
||||
|
||||
return { dataType: payloadType };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
import { sanitizeQueueName } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
|
||||
import type {
|
||||
LockedBackgroundWorker,
|
||||
QueueManager,
|
||||
QueueProperties,
|
||||
QueueValidationResult,
|
||||
TriggerTaskRequest,
|
||||
} from "../types";
|
||||
import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server";
|
||||
import type { RunEngine } from "~/v3/runEngine.server";
|
||||
import { env } from "~/env.server";
|
||||
import { tryCatch } from "@trigger.dev/core/v3";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
import { isInfrastructureError } from "~/utils/prismaErrors";
|
||||
import {
|
||||
createCache,
|
||||
createLRUMemoryStore,
|
||||
DefaultStatefulContext,
|
||||
Namespace,
|
||||
} from "@internal/cache";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import type { TaskMetadataCache, TaskMetadataEntry } from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import {
|
||||
recordTaskMetaResolve,
|
||||
type TaskMetaResolveSource,
|
||||
} from "~/services/taskMetadataCacheTelemetry.server";
|
||||
|
||||
// LRU cache for environment queue sizes to reduce Redis calls
|
||||
const queueSizeCache = singleton("queueSizeCache", () => {
|
||||
const ctx = new DefaultStatefulContext();
|
||||
const memory = createLRUMemoryStore(env.QUEUE_SIZE_CACHE_MAX_SIZE, "queue-size-cache");
|
||||
|
||||
return createCache({
|
||||
queueSize: new Namespace<number>(ctx, {
|
||||
stores: [memory],
|
||||
fresh: env.QUEUE_SIZE_CACHE_TTL_MS,
|
||||
stale: env.QUEUE_SIZE_CACHE_TTL_MS + 1000,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Extract the queue name from a queue option that may be:
|
||||
* - An object with a string `name` property: { name: "queue-name" }
|
||||
* - A double-wrapped object (bug case): { name: { name: "queue-name", ... } }
|
||||
*
|
||||
* This handles the case where the SDK accidentally double-wraps the queue
|
||||
* option when it's already an object with a name property.
|
||||
*/
|
||||
function extractQueueName(queue: { name?: unknown } | undefined): string | undefined {
|
||||
if (!queue?.name) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Normal case: queue.name is a string
|
||||
if (typeof queue.name === "string") {
|
||||
return queue.name;
|
||||
}
|
||||
|
||||
// Double-wrapped case: queue.name is an object with its own name property
|
||||
if (typeof queue.name === "object" && queue.name !== null && "name" in queue.name) {
|
||||
const innerName = (queue.name as { name: unknown }).name;
|
||||
if (typeof innerName === "string") {
|
||||
return innerName;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export class DefaultQueueManager implements QueueManager {
|
||||
private readonly replicaPrisma: PrismaClientOrTransaction;
|
||||
private readonly taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaClientOrTransaction,
|
||||
private readonly engine: RunEngine,
|
||||
replicaPrisma?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
this.replicaPrisma = replicaPrisma ?? prisma;
|
||||
this.taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
async resolveQueueProperties(
|
||||
request: TriggerTaskRequest,
|
||||
lockedBackgroundWorker?: LockedBackgroundWorker
|
||||
): Promise<QueueProperties> {
|
||||
let queueName: string;
|
||||
let lockedQueueId: string | undefined;
|
||||
let taskTtl: string | null | undefined;
|
||||
let taskKind: string | undefined;
|
||||
|
||||
// Determine queue name based on lockToVersion and provided options
|
||||
if (lockedBackgroundWorker) {
|
||||
// Task is locked to a specific worker version
|
||||
const specifiedQueueName = extractQueueName(request.body.options?.queue);
|
||||
|
||||
if (specifiedQueueName) {
|
||||
// A specific queue name is provided, validate it exists for the locked worker.
|
||||
// Pre-existing query — not cached because TaskQueue rows can be added or
|
||||
// removed independently of BackgroundWorkerTask, and a stale "queue exists"
|
||||
// claim would silently route to the wrong queue.
|
||||
const specifiedQueue = await this.prisma.taskQueue.findFirst({
|
||||
where: {
|
||||
name: specifiedQueueName,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
workers: { some: { id: lockedBackgroundWorker.id } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!specifiedQueue) {
|
||||
throw new ServiceValidationError(
|
||||
`Specified queue '${specifiedQueueName}' not found or not associated with locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
// Use the validated queue name directly
|
||||
queueName = specifiedQueue.name;
|
||||
lockedQueueId = specifiedQueue.id;
|
||||
|
||||
// Pull `triggerSource` (for `taskKind` annotation) and `ttl` from cache.
|
||||
// On cache hit this is 0 PG queries; on miss the helper falls back to
|
||||
// a BackgroundWorkerTask lookup and back-fills the cache.
|
||||
//
|
||||
// If the task slug isn't on this locked worker version, we tolerate
|
||||
// the missing row and fall through with `taskKind = undefined`
|
||||
// (coalesced to "STANDARD" downstream) and `taskTtl = undefined`.
|
||||
// This matches main's pre-PR behavior — the no-override branch below
|
||||
// still throws because there's no queue to route to in that case,
|
||||
// but here the caller already named the queue.
|
||||
const lockedMeta = await this.resolveLockedTaskMetadata(
|
||||
lockedBackgroundWorker.id,
|
||||
request.environment.id,
|
||||
request.taskId
|
||||
);
|
||||
|
||||
if (request.body.options?.ttl === undefined) {
|
||||
taskTtl = lockedMeta?.ttl ?? undefined;
|
||||
}
|
||||
taskKind = lockedMeta?.triggerSource;
|
||||
} else {
|
||||
// No queue override - resolve default queue + TTL + triggerSource via cache,
|
||||
// falling back to a single BackgroundWorkerTask lookup on miss.
|
||||
const lockedMeta = await this.resolveLockedTaskMetadata(
|
||||
lockedBackgroundWorker.id,
|
||||
request.environment.id,
|
||||
request.taskId
|
||||
);
|
||||
|
||||
if (!lockedMeta) {
|
||||
throw new ServiceValidationError(
|
||||
`Task '${request.taskId}' not found on locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
taskTtl = lockedMeta.ttl;
|
||||
|
||||
if (!lockedMeta.queueName) {
|
||||
// This case should ideally be prevented by earlier checks or schema constraints,
|
||||
// but handle it defensively.
|
||||
logger.error("Task found on locked version, but has no associated queue record", {
|
||||
taskId: request.taskId,
|
||||
workerId: lockedBackgroundWorker.id,
|
||||
version: lockedBackgroundWorker.version,
|
||||
});
|
||||
throw new ServiceValidationError(
|
||||
`Default queue configuration for task '${request.taskId}' missing on locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
// Use the task's default queue name
|
||||
queueName = lockedMeta.queueName;
|
||||
lockedQueueId = lockedMeta.queueId ?? undefined;
|
||||
taskKind = lockedMeta.triggerSource;
|
||||
}
|
||||
} else {
|
||||
// Task is not locked to a specific version, use regular logic
|
||||
if (request.body.options?.lockToVersion) {
|
||||
// This should only happen if the findFirst failed, indicating the version doesn't exist
|
||||
throw new ServiceValidationError(
|
||||
`Task locked to version '${request.body.options.lockToVersion}', but no worker found with that version.`
|
||||
);
|
||||
}
|
||||
|
||||
// Get queue name using the helper for non-locked case (handles provided name or finds default)
|
||||
const taskInfo = await this.getTaskQueueInfo(request);
|
||||
queueName = taskInfo.queueName;
|
||||
taskTtl = taskInfo.taskTtl;
|
||||
taskKind = taskInfo.taskKind;
|
||||
}
|
||||
|
||||
// Sanitize the final determined queue name once
|
||||
const sanitizedQueueName = sanitizeQueueName(queueName);
|
||||
|
||||
// Check that the queuename is not an empty string
|
||||
if (!sanitizedQueueName) {
|
||||
queueName = sanitizeQueueName(`task/${request.taskId}`); // Fallback if sanitization results in empty
|
||||
} else {
|
||||
queueName = sanitizedQueueName;
|
||||
}
|
||||
|
||||
return {
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
taskTtl,
|
||||
taskKind,
|
||||
};
|
||||
}
|
||||
|
||||
private async getTaskQueueInfo(
|
||||
request: TriggerTaskRequest
|
||||
): Promise<{ queueName: string; taskTtl?: string | null; taskKind?: string | undefined }> {
|
||||
const { taskId, environment, body } = request;
|
||||
const { queue } = body.options ?? {};
|
||||
|
||||
// Use extractQueueName to handle double-wrapped queue objects
|
||||
const overriddenQueueName = extractQueueName(queue);
|
||||
|
||||
const defaultQueueName = `task/${taskId}`;
|
||||
|
||||
// Resolve the current worker's task metadata via cache (HGET on warm path,
|
||||
// BackgroundWorkerTask findFirst + cache back-fill on miss). When this hits,
|
||||
// both the queue-override + TTL caller and the default-queue caller satisfy
|
||||
// their full result without any database query.
|
||||
const meta = await this.resolveCurrentTaskMetadata(environment, taskId);
|
||||
|
||||
if (overriddenQueueName) {
|
||||
// Caller already named the queue. We only need triggerSource (for taskKind)
|
||||
// and ttl (for the call site to coalesce against body.options.ttl).
|
||||
return {
|
||||
queueName: overriddenQueueName,
|
||||
taskTtl: meta?.ttl ?? undefined,
|
||||
taskKind: meta?.triggerSource,
|
||||
};
|
||||
}
|
||||
|
||||
if (!meta) {
|
||||
logger.debug("Failed to get queue name: No worker or task found", {
|
||||
taskId,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
return { queueName: defaultQueueName, taskTtl: undefined };
|
||||
}
|
||||
|
||||
if (!meta.queueName) {
|
||||
logger.debug("Failed to get queue name: No queue found", {
|
||||
taskId,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
return { queueName: defaultQueueName, taskTtl: meta.ttl, taskKind: meta.triggerSource };
|
||||
}
|
||||
|
||||
return { queueName: meta.queueName, taskTtl: meta.ttl, taskKind: meta.triggerSource };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve task metadata for a locked-version trigger. Reads from the
|
||||
* `task-meta:by-worker:{workerId}` Redis hash; falls back to a single
|
||||
* BackgroundWorkerTask findFirst on miss and back-fills the cache.
|
||||
*
|
||||
* Returns null when no BackgroundWorkerTask row exists.
|
||||
*/
|
||||
private async resolveLockedTaskMetadata(
|
||||
workerId: string,
|
||||
environmentId: string,
|
||||
slug: string
|
||||
): Promise<TaskMetadataEntry | null> {
|
||||
const cached = await this.taskMetaCache.getByWorker(workerId, slug);
|
||||
if (cached) {
|
||||
recordTaskMetaResolve("locked", "cache");
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Cache miss. Read the row from the replica first; if the replica comes
|
||||
// back empty, re-check the writer before concluding the task is missing.
|
||||
// The locked worker itself was just resolved on the writer (see
|
||||
// triggerTask.server.ts), so a replica that returns no row here is stale,
|
||||
// not authoritative. Trusting a stale-replica negative throws a
|
||||
// non-retryable "not found on locked version" for a task that is in fact
|
||||
// registered. The writer read only runs on this rare miss-then-empty path,
|
||||
// never on the hot path.
|
||||
let row = await this.findLockedTaskRow(this.replicaPrisma, workerId, environmentId, slug);
|
||||
let source: TaskMetaResolveSource = "replica";
|
||||
|
||||
if (!row && this.replicaPrisma !== this.prisma) {
|
||||
row = await this.findLockedTaskRow(this.prisma, workerId, environmentId, slug);
|
||||
|
||||
if (row) {
|
||||
source = "writer";
|
||||
logger.warn("Locked task metadata missing on replica but found on writer", {
|
||||
workerId,
|
||||
environmentId,
|
||||
slug,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
recordTaskMetaResolve("locked", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
recordTaskMetaResolve("locked", source);
|
||||
|
||||
const entry: TaskMetadataEntry = {
|
||||
slug,
|
||||
ttl: row.ttl,
|
||||
triggerSource: row.triggerSource,
|
||||
queueId: row.queue?.id ?? null,
|
||||
queueName: row.queue?.name ?? "",
|
||||
};
|
||||
|
||||
// Fire-and-forget back-fill — `setByWorker` upserts the single field and
|
||||
// refreshes the hash TTL. Errors are logged inside the cache and swallowed.
|
||||
void this.taskMetaCache.setByWorker(workerId, entry);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
private findLockedTaskRow(
|
||||
client: PrismaClientOrTransaction,
|
||||
workerId: string,
|
||||
environmentId: string,
|
||||
slug: string
|
||||
) {
|
||||
return client.backgroundWorkerTask.findFirst({
|
||||
where: { workerId, runtimeEnvironmentId: environmentId, slug },
|
||||
select: {
|
||||
ttl: true,
|
||||
triggerSource: true,
|
||||
queue: { select: { id: true, name: true } },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve task metadata for a non-locked trigger. Reads from the
|
||||
* `task-meta:env:{envId}` Redis hash; falls back to
|
||||
* findCurrentWorkerFromEnvironment + a single BackgroundWorkerTask findFirst
|
||||
* on miss and back-fills both keyspaces.
|
||||
*
|
||||
* Returns null when no current worker or task can be resolved.
|
||||
*/
|
||||
private async resolveCurrentTaskMetadata(
|
||||
environment: AuthenticatedEnvironment,
|
||||
slug: string
|
||||
): Promise<TaskMetadataEntry | null> {
|
||||
const cached = await this.taskMetaCache.getCurrent(environment.id, slug);
|
||||
if (cached) {
|
||||
recordTaskMetaResolve("current", "cache");
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Cold cache: discover the current worker for the env. Replica is fine —
|
||||
// the adjacent BackgroundWorkerTask lookup below uses `replicaPrisma` too
|
||||
// (replica lag for "just deployed" is bounded the same way for both
|
||||
// queries; reading from the writer here would only widen the window).
|
||||
const worker = await findCurrentWorkerFromEnvironment(environment, this.replicaPrisma);
|
||||
if (!worker) {
|
||||
recordTaskMetaResolve("current", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = await this.replicaPrisma.backgroundWorkerTask.findFirst({
|
||||
where: { workerId: worker.id, runtimeEnvironmentId: environment.id, slug },
|
||||
select: {
|
||||
ttl: true,
|
||||
triggerSource: true,
|
||||
queue: { select: { id: true, name: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!row) {
|
||||
recordTaskMetaResolve("current", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
recordTaskMetaResolve("current", "replica");
|
||||
|
||||
const entry: TaskMetadataEntry = {
|
||||
slug,
|
||||
ttl: row.ttl,
|
||||
triggerSource: row.triggerSource,
|
||||
queueId: row.queue?.id ?? null,
|
||||
queueName: row.queue?.name ?? "",
|
||||
};
|
||||
|
||||
// Fire-and-forget back-fill — atomically upserts the slug into both
|
||||
// keyspaces so a subsequent locked-or-not trigger hits the cache. The
|
||||
// env-keyspace TTL is preserved (promotion owns it); the by-worker TTL
|
||||
// is refreshed (sliding window keeps active workers warm).
|
||||
void this.taskMetaCache.setByCurrentWorker(environment.id, worker.id, entry);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
async validateQueueLimits(
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string,
|
||||
itemsToAdd?: number
|
||||
): Promise<QueueValidationResult> {
|
||||
const queueSizeGuard = await guardQueueSizeLimitsForQueue(
|
||||
this.engine,
|
||||
environment,
|
||||
queueName,
|
||||
itemsToAdd
|
||||
);
|
||||
|
||||
logger.debug("Queue size guard result", {
|
||||
queueSizeGuard,
|
||||
queueName,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
type: environment.type,
|
||||
organization: environment.organization,
|
||||
project: environment.project,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
ok: queueSizeGuard.isWithinLimits,
|
||||
maximumSize: queueSizeGuard.maximumSize ?? 0,
|
||||
queueSize: queueSizeGuard.queueSize ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
async getWorkerQueue(
|
||||
environment: AuthenticatedEnvironment,
|
||||
regionOverride?: string
|
||||
): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined> {
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
return { masterQueue: environment.id, enableFastPath: true };
|
||||
}
|
||||
|
||||
const workerGroupService = new WorkerGroupService({
|
||||
prisma: this.prisma,
|
||||
engine: this.engine,
|
||||
});
|
||||
|
||||
const [error, workerGroup] = await tryCatch(
|
||||
workerGroupService.getDefaultWorkerGroupForProject({
|
||||
projectId: environment.projectId,
|
||||
regionOverride,
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
// getDefaultWorkerGroupForProject queries the writer DB. A Prisma
|
||||
// infrastructure error (e.g. P1001 "Can't reach database server", whose
|
||||
// message carries the DB hostname) must NOT be promoted into a
|
||||
// client-facing ServiceValidationError: that leaks internal infra detail
|
||||
// to the API client (the SDK echoes it into the run view) and
|
||||
// mis-classifies a transient outage as a non-retryable 422. Let it
|
||||
// propagate to the route's generic 500 handler (scrubbed + retryable);
|
||||
// only wrap genuine domain failures.
|
||||
if (isInfrastructureError(error)) {
|
||||
throw error;
|
||||
}
|
||||
throw new ServiceValidationError(error.message);
|
||||
}
|
||||
|
||||
if (!workerGroup) {
|
||||
throw new ServiceValidationError("No worker group found");
|
||||
}
|
||||
|
||||
return {
|
||||
masterQueue: workerGroup.masterQueue,
|
||||
enableFastPath: workerGroup.enableFastPath,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function getMaximumSizeForEnvironment(
|
||||
environment: AuthenticatedEnvironment
|
||||
): number | undefined {
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
return environment.organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE;
|
||||
} else {
|
||||
return environment.organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
async function guardQueueSizeLimitsForQueue(
|
||||
engine: RunEngine,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string,
|
||||
itemsToAdd: number = 1
|
||||
) {
|
||||
const maximumSize = getMaximumSizeForEnvironment(environment);
|
||||
|
||||
if (typeof maximumSize === "undefined") {
|
||||
return { isWithinLimits: true };
|
||||
}
|
||||
|
||||
const queueSize = await getCachedQueueSize(engine, environment, queueName);
|
||||
const projectedSize = queueSize + itemsToAdd;
|
||||
|
||||
return {
|
||||
isWithinLimits: projectedSize <= maximumSize,
|
||||
maximumSize,
|
||||
queueSize,
|
||||
};
|
||||
}
|
||||
|
||||
async function getCachedQueueSize(
|
||||
engine: RunEngine,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string
|
||||
): Promise<number> {
|
||||
if (!env.QUEUE_SIZE_CACHE_ENABLED) {
|
||||
return engine.lengthOfQueue(environment, queueName);
|
||||
}
|
||||
|
||||
const cacheKey = `${environment.id}:${queueName}`;
|
||||
const result = await queueSizeCache.queueSize.swr(cacheKey, async () => {
|
||||
return engine.lengthOfQueue(environment, queueName);
|
||||
});
|
||||
|
||||
return result.val ?? 0;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import {
|
||||
$replica as defaultLegacyReplica,
|
||||
runOpsNewPrisma as defaultNewPrimary,
|
||||
runOpsNewReplica as defaultNewClient,
|
||||
runOpsSplitReadEnabled as defaultSplitReadEnabled,
|
||||
} from "~/db.server";
|
||||
import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server";
|
||||
|
||||
type ResolveWaitpointDeps = {
|
||||
newClient?: PrismaReplicaClient;
|
||||
legacyReplica?: PrismaReplicaClient;
|
||||
newPrimary?: PrismaReplicaClient;
|
||||
splitEnabled?: boolean;
|
||||
isPastRetention?: (id: string) => boolean;
|
||||
};
|
||||
|
||||
// Safe defaults matching the deps `complete`/`callback` pass, so a bare caller still fans
|
||||
// out to the dedicated run-ops replica (NEW-resident waitpoints) before control-plane.
|
||||
export type ResolveWaitpointReadThroughDefaults = {
|
||||
newClient: PrismaReplicaClient;
|
||||
legacyReplica: PrismaReplicaClient;
|
||||
newPrimary: PrismaReplicaClient;
|
||||
splitEnabled: boolean;
|
||||
};
|
||||
|
||||
const productionDefaults: ResolveWaitpointReadThroughDefaults = {
|
||||
newClient: defaultNewClient,
|
||||
legacyReplica: defaultLegacyReplica,
|
||||
newPrimary: defaultNewPrimary as unknown as PrismaReplicaClient,
|
||||
splitEnabled: defaultSplitReadEnabled,
|
||||
};
|
||||
|
||||
export async function resolveWaitpointThroughReadThrough<T>(opts: {
|
||||
waitpointId: string;
|
||||
environmentId: string;
|
||||
read: (client: PrismaReplicaClient) => Promise<T | null>;
|
||||
deps?: ResolveWaitpointDeps;
|
||||
defaults?: ResolveWaitpointReadThroughDefaults;
|
||||
}): Promise<T | null> {
|
||||
const defaults = opts.defaults ?? productionDefaults;
|
||||
|
||||
const splitEnabled = opts.deps?.splitEnabled ?? defaults.splitEnabled;
|
||||
|
||||
const result = await readThroughRun({
|
||||
runId: opts.waitpointId,
|
||||
environmentId: opts.environmentId,
|
||||
readNew: (client) => opts.read(client),
|
||||
readLegacy: (replica) => opts.read(replica),
|
||||
deps: {
|
||||
splitEnabled,
|
||||
newClient: opts.deps?.newClient ?? defaults.newClient,
|
||||
legacyReplica: opts.deps?.legacyReplica ?? defaults.legacyReplica,
|
||||
isPastRetention: opts.deps?.isPastRetention,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.source === "new" || result.source === "legacy-replica") {
|
||||
return result.value;
|
||||
}
|
||||
// past-retention is an intentional not-found: the token is gone.
|
||||
if (result.source === "past-retention") {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read-your-writes fallback for a token completed immediately after mint, before it replicated:
|
||||
// re-read from the run-ops PRIMARY only. We deliberately never read the control-plane/legacy
|
||||
// primary here (that is the load the replica-only read-through exists to shed), so a legacy-resident
|
||||
// token that misses its replica stays a miss and the caller retries, rather than adding primary load.
|
||||
const fromNewPrimary = await opts.read(opts.deps?.newPrimary ?? defaults.newPrimary);
|
||||
if (fromNewPrimary != null) {
|
||||
return fromNewPrimary;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { SemanticInternalAttributes } from "@trigger.dev/core/v3/semanticInternalAttributes";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import type { IEventRepository } from "~/v3/eventRepository/eventRepository.types";
|
||||
import { getEventRepository } from "~/v3/eventRepository/index.server";
|
||||
import type { TracedEventSpan, TraceEventConcern, TriggerTaskRequest } from "../types";
|
||||
|
||||
export class DefaultTraceEventsConcern implements TraceEventConcern {
|
||||
async #getEventRepository(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined
|
||||
): Promise<{ repository: IEventRepository; store: string }> {
|
||||
return await getEventRepository(
|
||||
request.environment.organization.id,
|
||||
request.environment.organization.featureFlags as Record<string, unknown>,
|
||||
parentStore
|
||||
);
|
||||
}
|
||||
|
||||
async traceRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
request.taskId,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {},
|
||||
style: {
|
||||
icon: request.options?.customIcon ?? "task",
|
||||
},
|
||||
},
|
||||
incomplete: true,
|
||||
immediate: true,
|
||||
startTime: request.options?.overrideCreatedAt
|
||||
? BigInt(request.options.overrideCreatedAt.getTime()) * BigInt(1000000)
|
||||
: undefined,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async traceIdempotentRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
idempotencyKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { existingRun, idempotencyKey, incomplete, isError } = options;
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
`${request.taskId} (cached)`,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {
|
||||
[SemanticInternalAttributes.ORIGINAL_RUN_ID]: existingRun.friendlyId,
|
||||
},
|
||||
style: {
|
||||
icon: "task-cached",
|
||||
},
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
incomplete,
|
||||
isError,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
//log a message
|
||||
await repository.recordEvent(
|
||||
`There's an existing run for idempotencyKey: ${idempotencyKey}`,
|
||||
{
|
||||
taskSlug: request.taskId,
|
||||
environment: request.environment,
|
||||
attributes: {
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
context: request.options?.traceContext,
|
||||
parentId: event.spanId,
|
||||
}
|
||||
);
|
||||
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async traceDebouncedRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
debounceKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { existingRun, debounceKey, incomplete, isError } = options;
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
`${request.taskId} (debounced)`,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {
|
||||
[SemanticInternalAttributes.ORIGINAL_RUN_ID]: existingRun.friendlyId,
|
||||
},
|
||||
style: {
|
||||
icon: "task-cached",
|
||||
},
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
incomplete,
|
||||
isError,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
// Log a message about the debounced trigger
|
||||
await repository.recordEvent(`Debounced: using existing run with key "${debounceKey}"`, {
|
||||
taskSlug: request.taskId,
|
||||
environment: request.environment,
|
||||
attributes: {
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
context: request.options?.traceContext,
|
||||
parentId: event.spanId,
|
||||
});
|
||||
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { env } from "~/env.server";
|
||||
import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
function packetExtensionForDataType(dataType: string): string {
|
||||
switch (dataType) {
|
||||
case "application/json":
|
||||
case "application/super+json":
|
||||
return "json";
|
||||
case "text/plain":
|
||||
return "txt";
|
||||
default:
|
||||
return "txt";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloads large waitpoint completion payloads to object store (same threshold and
|
||||
* upload path pattern as DefaultPayloadProcessor). Object key prefix should use the
|
||||
* waitpoint friendly id folder, e.g. `${WaitpointId.toFriendlyId(internalId)}/token`.
|
||||
* Replaces no-op conditionallyExportPacket usage in webapp routes where apiClientManager is unset.
|
||||
*/
|
||||
export async function processWaitpointCompletionPacket(
|
||||
packet: IOPacket,
|
||||
environment: AuthenticatedEnvironment,
|
||||
pathPrefix: string
|
||||
): Promise<IOPacket> {
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading, size: _size } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${pathPrefix}.${packetExtensionForDataType(packet.dataType)}`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
logger.error("Failed to upload large waitpoint to object store", {
|
||||
error: uploadError,
|
||||
filename,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
throw new ServiceValidationError("Failed to upload large waitpoint to object store", 500);
|
||||
}
|
||||
|
||||
return {
|
||||
data: uploadedFilename!,
|
||||
dataType: "application/store",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { WorkerQueueClass } from "@trigger.dev/core/v3/workers";
|
||||
import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags";
|
||||
|
||||
/**
|
||||
* Suffix appended to a region's worker queue name to route scheduled-lineage
|
||||
* runs onto their own Redis list (e.g. `us-nyc-3` -> `us-nyc-3:scheduled`). A
|
||||
* dedicated consumer fleet dequeues the suffixed list so the top-of-hour
|
||||
* scheduled-cron herd can't starve standard/agent run startup. The worker queue
|
||||
* name is opaque everywhere downstream (it's only ever `:`-joined into a Redis
|
||||
* key and persisted on the run), so encoding the class in the suffix needs no
|
||||
* Lua, envelope, or resolver changes.
|
||||
*/
|
||||
export const SCHEDULED_WORKER_QUEUE_SUFFIX = ":scheduled";
|
||||
|
||||
/**
|
||||
* Recover the base region a worker queue belongs to by stripping any split
|
||||
* suffix (e.g. `us-nyc-3:scheduled` -> `us-nyc-3`). Region/masterQueue names are
|
||||
* either `<name>` or `<projectId>-<name>` and never contain a colon, so the
|
||||
* region is everything before the first `:`. Use this wherever a worker queue is
|
||||
* read as a region — for display, filtering, or as a region override — so
|
||||
* scheduled-split runs group under their real region instead of a phantom one.
|
||||
* Idempotent; returns the input unchanged when there's no suffix. A nullish
|
||||
* worker queue (e.g. from a synthetic run snapshot) passes straight through.
|
||||
*/
|
||||
export function baseWorkerQueue(workerQueue: string): string;
|
||||
export function baseWorkerQueue(workerQueue: string | null | undefined): string | null | undefined;
|
||||
export function baseWorkerQueue(workerQueue: string | null | undefined): string | null | undefined {
|
||||
if (workerQueue == null) {
|
||||
return workerQueue;
|
||||
}
|
||||
|
||||
const colon = workerQueue.indexOf(":");
|
||||
return colon === -1 ? workerQueue : workerQueue.slice(0, colon);
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing region for read surfaces: the explicit geo region if set, else the
|
||||
* region derived from the worker queue, else undefined. Use everywhere a run's
|
||||
* region is displayed so an empty queue never surfaces as `""` and all surfaces
|
||||
* agree. Not for query keys — those want the raw worker queue, not this fallback.
|
||||
*/
|
||||
export function regionForDisplay(
|
||||
region: string | null | undefined,
|
||||
workerQueue: string | null | undefined
|
||||
): string | undefined {
|
||||
return region || (workerQueue ? baseWorkerQueue(workerQueue) : undefined);
|
||||
}
|
||||
|
||||
/** `TriggerSource` value used for runs originating from a schedule. */
|
||||
const SCHEDULE_TRIGGER_SOURCE = "schedule";
|
||||
|
||||
/**
|
||||
* Resolve whether the scheduled worker-queue split is enabled for a run, reading
|
||||
* only the in-memory org feature-flags JSON (already loaded on the authenticated
|
||||
* environment) — never a DB query, so it is safe on the trigger hot path.
|
||||
*
|
||||
* Precedence: a per-org override wins in BOTH directions; the global default is
|
||||
* used only when the org has not set the flag.
|
||||
*/
|
||||
export function resolveScheduledQueueSplitEnabled({
|
||||
orgFeatureFlags,
|
||||
globalDefault,
|
||||
}: {
|
||||
orgFeatureFlags: Record<string, unknown> | null | undefined;
|
||||
globalDefault: boolean;
|
||||
}): boolean {
|
||||
const override = orgFeatureFlags?.[FEATURE_FLAG.workerQueueScheduledSplitEnabled];
|
||||
|
||||
if (override !== undefined) {
|
||||
const parsed =
|
||||
FeatureFlagCatalog[FEATURE_FLAG.workerQueueScheduledSplitEnabled].safeParse(override);
|
||||
|
||||
if (parsed.success) {
|
||||
return parsed.data;
|
||||
}
|
||||
}
|
||||
|
||||
return globalDefault;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the worker queue a run should be enqueued onto. Runs in a scheduled
|
||||
* lineage (`rootTriggerSource === "schedule"`, which propagates from a scheduled
|
||||
* root down to every descendant) route to the suffixed list when the split is
|
||||
* enabled; everything else is unchanged. Idempotent — never double-suffixes.
|
||||
*/
|
||||
export function workerQueueForRun({
|
||||
workerQueue,
|
||||
rootTriggerSource,
|
||||
splitEnabled,
|
||||
}: {
|
||||
workerQueue: string;
|
||||
rootTriggerSource: string | undefined;
|
||||
splitEnabled: boolean;
|
||||
}): string {
|
||||
if (
|
||||
!splitEnabled ||
|
||||
rootTriggerSource !== SCHEDULE_TRIGGER_SOURCE ||
|
||||
workerQueue.endsWith(SCHEDULED_WORKER_QUEUE_SUFFIX)
|
||||
) {
|
||||
return workerQueue;
|
||||
}
|
||||
|
||||
return `${workerQueue}${SCHEDULED_WORKER_QUEUE_SUFFIX}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumer-side counterpart to {@link workerQueueForRun}: given a worker's base
|
||||
* (region) queue and the requested queue class, return the worker queue to
|
||||
* dequeue from. `"scheduled"` targets the suffixed list; anything else is the
|
||||
* base queue. The server always derives this from the authenticated worker's
|
||||
* own `masterQueue`, so a token can only ever reach its own region's queues.
|
||||
* Idempotent — never double-suffixes.
|
||||
*/
|
||||
export function workerQueueForClass(
|
||||
masterQueue: string,
|
||||
queueClass: WorkerQueueClass | undefined
|
||||
): string {
|
||||
if (queueClass === "scheduled" && !masterQueue.endsWith(SCHEDULED_WORKER_QUEUE_SUFFIX)) {
|
||||
return `${masterQueue}${SCHEDULED_WORKER_QUEUE_SUFFIX}`;
|
||||
}
|
||||
|
||||
return masterQueue;
|
||||
}
|
||||
|
||||
export function parseDisabledWorkerQueues(raw: string | undefined): Set<string> {
|
||||
return new Set(
|
||||
(raw ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
);
|
||||
}
|
||||
|
||||
export function matchesDisabledWorkerQueue(
|
||||
workerQueue: string,
|
||||
disabledWorkerQueues: ReadonlySet<string>
|
||||
): boolean {
|
||||
if (disabledWorkerQueues.size === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
disabledWorkerQueues.has(workerQueue) || disabledWorkerQueues.has(baseWorkerQueue(workerQueue))
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user