chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,35 @@
|
||||
import { Ratelimit } from "@upstash/ratelimit";
|
||||
import type { GlobalRateLimiter } from "@trigger.dev/redis-worker";
|
||||
import { RateLimiter } from "~/services/rateLimiter.server";
|
||||
|
||||
/**
|
||||
* Creates a global rate limiter for the batch queue that limits
|
||||
* the maximum number of items processed per second across all consumers.
|
||||
*
|
||||
* Uses a token bucket algorithm where:
|
||||
* - `itemsPerSecond` tokens are available per second
|
||||
* - The bucket can hold up to `itemsPerSecond` tokens (burst capacity)
|
||||
*
|
||||
* @param itemsPerSecond - Maximum items to process per second
|
||||
* @returns A GlobalRateLimiter compatible with FairQueue
|
||||
*/
|
||||
export function createBatchGlobalRateLimiter(itemsPerSecond: number): GlobalRateLimiter {
|
||||
const limiter = new RateLimiter({
|
||||
keyPrefix: "batch-queue-global",
|
||||
// Token bucket: refills `itemsPerSecond` tokens every second
|
||||
// Bucket capacity is also `itemsPerSecond` (allows burst up to limit)
|
||||
limiter: Ratelimit.tokenBucket(itemsPerSecond, "1 s", itemsPerSecond),
|
||||
logSuccess: false,
|
||||
logFailure: true,
|
||||
});
|
||||
|
||||
return {
|
||||
async limit() {
|
||||
const result = await limiter.limit("global");
|
||||
return {
|
||||
allowed: result.success,
|
||||
resetAt: result.reset,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { z } from "zod";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
createLimiterFromConfig,
|
||||
RateLimiterConfig,
|
||||
} from "~/services/authorizationRateLimitMiddleware.server";
|
||||
import type { Duration } from "~/services/rateLimiter.server";
|
||||
import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
const BatchLimitsConfig = z.object({
|
||||
processingConcurrency: z.number().int().default(env.BATCH_CONCURRENCY_LIMIT_DEFAULT),
|
||||
});
|
||||
|
||||
/**
|
||||
* Batch limits configuration for a plan type
|
||||
*/
|
||||
export type BatchLimitsConfig = z.infer<typeof BatchLimitsConfig>;
|
||||
|
||||
const batchLimitsRedisClient = singleton("batchLimitsRedisClient", createBatchLimitsRedisClient);
|
||||
|
||||
function createBatchLimitsRedisClient() {
|
||||
const redisClient = createRedisRateLimitClient({
|
||||
port: env.RATE_LIMIT_REDIS_PORT,
|
||||
host: env.RATE_LIMIT_REDIS_HOST,
|
||||
username: env.RATE_LIMIT_REDIS_USERNAME,
|
||||
password: env.RATE_LIMIT_REDIS_PASSWORD,
|
||||
tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
});
|
||||
|
||||
return redisClient;
|
||||
}
|
||||
|
||||
// Just the org fields this module reads. Compatible with both the full
|
||||
// Prisma `Organization` payload and the slim `AuthenticatedEnvironment`
|
||||
// `["organization"]` shape (when passed `batchRateLimitConfig` /
|
||||
// `batchQueueConcurrencyConfig` as `unknown`).
|
||||
type OrganizationForBatchLimits = {
|
||||
batchRateLimitConfig?: unknown;
|
||||
batchQueueConcurrencyConfig?: unknown;
|
||||
};
|
||||
|
||||
function createOrganizationRateLimiter(organization: OrganizationForBatchLimits): RateLimiter {
|
||||
const limiterConfig = resolveBatchRateLimitConfig(organization.batchRateLimitConfig);
|
||||
|
||||
const limiter = createLimiterFromConfig(limiterConfig);
|
||||
|
||||
return new RateLimiter({
|
||||
redisClient: batchLimitsRedisClient,
|
||||
keyPrefix: "ratelimit:batch",
|
||||
limiter,
|
||||
logSuccess: false,
|
||||
logFailure: true,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveBatchRateLimitConfig(batchRateLimitConfig?: unknown): RateLimiterConfig {
|
||||
const defaultRateLimiterConfig: RateLimiterConfig = {
|
||||
type: "tokenBucket",
|
||||
refillRate: env.BATCH_RATE_LIMIT_REFILL_RATE,
|
||||
interval: env.BATCH_RATE_LIMIT_REFILL_INTERVAL as Duration,
|
||||
maxTokens: env.BATCH_RATE_LIMIT_MAX,
|
||||
};
|
||||
|
||||
if (!batchRateLimitConfig) {
|
||||
return defaultRateLimiterConfig;
|
||||
}
|
||||
|
||||
const parsedBatchRateLimitConfig = RateLimiterConfig.safeParse(batchRateLimitConfig);
|
||||
|
||||
if (!parsedBatchRateLimitConfig.success) {
|
||||
return defaultRateLimiterConfig;
|
||||
}
|
||||
|
||||
return parsedBatchRateLimitConfig.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the rate limiter and limits for an organization.
|
||||
* Internally looks up the plan type, but doesn't expose it to callers.
|
||||
*/
|
||||
export async function getBatchLimits(
|
||||
organization: OrganizationForBatchLimits
|
||||
): Promise<{ rateLimiter: RateLimiter; config: BatchLimitsConfig }> {
|
||||
const rateLimiter = createOrganizationRateLimiter(organization);
|
||||
const config = resolveBatchLimitsConfig(organization.batchQueueConcurrencyConfig);
|
||||
return { rateLimiter, config };
|
||||
}
|
||||
|
||||
function resolveBatchLimitsConfig(batchLimitsConfig?: unknown): BatchLimitsConfig {
|
||||
const defaultLimitsConfig: BatchLimitsConfig = {
|
||||
processingConcurrency: env.BATCH_CONCURRENCY_LIMIT_DEFAULT,
|
||||
};
|
||||
|
||||
if (!batchLimitsConfig) {
|
||||
return defaultLimitsConfig;
|
||||
}
|
||||
|
||||
const parsedBatchLimitsConfig = BatchLimitsConfig.safeParse(batchLimitsConfig);
|
||||
|
||||
if (!parsedBatchLimitsConfig.success) {
|
||||
return defaultLimitsConfig;
|
||||
}
|
||||
|
||||
return parsedBatchLimitsConfig.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when batch rate limit is exceeded.
|
||||
* Contains information for constructing a proper 429 response.
|
||||
*/
|
||||
export class BatchRateLimitExceededError extends Error {
|
||||
constructor(
|
||||
public readonly limit: number,
|
||||
public readonly remaining: number,
|
||||
public readonly resetAt: Date,
|
||||
public readonly itemCount: number
|
||||
) {
|
||||
super(`Batch rate limit exceeded. Limit resets at ${resetAt.toISOString()}`);
|
||||
this.name = "BatchRateLimitExceededError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import pRetry from "p-retry";
|
||||
import { env } from "~/env.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { hasObjectStoreClient, uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { startActiveSpan } from "~/v3/tracer.server";
|
||||
|
||||
export type BatchPayloadProcessResult = {
|
||||
/** The processed payload - either the original or an R2 path */
|
||||
payload: unknown;
|
||||
/** The payload type - "application/store" if offloaded to R2 */
|
||||
payloadType: string;
|
||||
/** Whether the payload was offloaded to R2 */
|
||||
wasOffloaded: boolean;
|
||||
/** Size of the payload in bytes */
|
||||
size: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* BatchPayloadProcessor handles payload offloading for batch items.
|
||||
*
|
||||
* When a batch item's payload exceeds the configured threshold, it's uploaded
|
||||
* to object storage (R2) and the payload is replaced with the storage path.
|
||||
* This aligns with how single task triggers work via DefaultPayloadProcessor.
|
||||
*
|
||||
* Path format: batch_{batchId}/item_{index}/payload.json
|
||||
*/
|
||||
export class BatchPayloadProcessor {
|
||||
/**
|
||||
* Check if object storage is available for payload offloading.
|
||||
* If not available, large payloads will be stored inline (which may fail for very large payloads).
|
||||
*/
|
||||
isObjectStoreAvailable(): boolean {
|
||||
return hasObjectStoreClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a batch item payload, offloading to R2 if it exceeds the threshold.
|
||||
*
|
||||
* @param payload - The raw payload from the batch item
|
||||
* @param payloadType - The payload type (e.g., "application/json")
|
||||
* @param batchId - The batch ID (internal format)
|
||||
* @param itemIndex - The item index within the batch
|
||||
* @param environment - The authenticated environment for R2 path construction
|
||||
* @returns The processed result with potentially offloaded payload
|
||||
*/
|
||||
async process(
|
||||
payload: unknown,
|
||||
payloadType: string,
|
||||
batchId: string,
|
||||
itemIndex: number,
|
||||
environment: AuthenticatedEnvironment
|
||||
): Promise<BatchPayloadProcessResult> {
|
||||
return startActiveSpan("BatchPayloadProcessor.process()", async (span) => {
|
||||
span.setAttribute("batchId", batchId);
|
||||
span.setAttribute("itemIndex", itemIndex);
|
||||
span.setAttribute("payloadType", payloadType);
|
||||
|
||||
// Create the packet for size checking
|
||||
const packet = this.#createPayloadPacket(payload, payloadType);
|
||||
|
||||
if (!packet.data) {
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const threshold = env.BATCH_PAYLOAD_OFFLOAD_THRESHOLD ?? env.TASK_PAYLOAD_OFFLOAD_THRESHOLD;
|
||||
const { needsOffloading, size } = packetRequiresOffloading(packet, threshold);
|
||||
|
||||
span.setAttribute("payloadSize", size);
|
||||
span.setAttribute("needsOffloading", needsOffloading);
|
||||
span.setAttribute("threshold", threshold);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size,
|
||||
};
|
||||
}
|
||||
|
||||
// Check if object store is available
|
||||
if (!this.isObjectStoreAvailable()) {
|
||||
logger.warn("Payload exceeds threshold but object store is not available", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
size,
|
||||
threshold,
|
||||
});
|
||||
|
||||
// Return without offloading - the payload will be stored inline
|
||||
// This may fail downstream for very large payloads
|
||||
return {
|
||||
payload,
|
||||
payloadType,
|
||||
wasOffloaded: false,
|
||||
size,
|
||||
};
|
||||
}
|
||||
|
||||
// Upload to object store, retrying on transient network errors
|
||||
const { data: packetData, dataType: packetDataType } = packet;
|
||||
const filename = `batch_${batchId}/item_${itemIndex}/payload.json`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
pRetry(
|
||||
() =>
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packetData,
|
||||
packetDataType,
|
||||
environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
),
|
||||
{
|
||||
retries: 3,
|
||||
minTimeout: 500,
|
||||
maxTimeout: 2000,
|
||||
factor: 2,
|
||||
onFailedAttempt: (error) => {
|
||||
logger.warn("Batch item payload upload to object store failed, retrying", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
attempt: error.attemptNumber,
|
||||
retriesLeft: error.retriesLeft,
|
||||
error: error.message,
|
||||
});
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
logger.error("Failed to upload batch item payload to object store after retries", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
error: uploadError.message,
|
||||
});
|
||||
|
||||
throw new Error(`Failed to upload large payload to object store: ${uploadError.message}`);
|
||||
}
|
||||
|
||||
logger.debug("Batch item payload offloaded to object store", {
|
||||
batchId,
|
||||
itemIndex,
|
||||
filename: uploadedFilename,
|
||||
size,
|
||||
});
|
||||
|
||||
span.setAttribute("wasOffloaded", true);
|
||||
span.setAttribute("offloadPath", uploadedFilename);
|
||||
|
||||
return {
|
||||
payload: uploadedFilename!,
|
||||
payloadType: "application/store",
|
||||
wasOffloaded: true,
|
||||
size,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOPacket from payload for size checking.
|
||||
*/
|
||||
#createPayloadPacket(payload: unknown, payloadType: string): IOPacket {
|
||||
if (payloadType === "application/json") {
|
||||
// Payload from SDK is already serialized as a string - use directly
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: "application/json" };
|
||||
}
|
||||
// Non-string payloads (e.g., direct API calls with objects) need serialization
|
||||
return { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
}
|
||||
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: payloadType };
|
||||
}
|
||||
|
||||
// For other types, try to stringify
|
||||
try {
|
||||
return { data: JSON.stringify(payload), dataType: payloadType };
|
||||
} catch {
|
||||
return { dataType: payloadType };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { hashBucket } from "~/utils/computeBucket";
|
||||
|
||||
/** Subset of the global flags snapshot this resolver reads. */
|
||||
export type ComputeMigrationFlags = {
|
||||
computeMigrationEnabled?: boolean;
|
||||
computeMigrationFreePercentage?: number;
|
||||
computeMigrationPaidPercentage?: number;
|
||||
};
|
||||
|
||||
type MigrationDecisionInput = {
|
||||
planType: string | undefined;
|
||||
orgId: string;
|
||||
orgFeatureFlags: Record<string, unknown> | null | undefined;
|
||||
flags: ComputeMigrationFlags | undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether this org should run on the compute backing. Shared by the trigger-time
|
||||
* transform and the deploy-time template decision so a migrated org always gets a
|
||||
* compute template. Precedence: per-org override (both directions) wins; otherwise
|
||||
* global enable + the plan's percentage bucket. Enterprise and unknown plans are
|
||||
* never enrolled by percentage (override only). The sole opt-out is the per-org
|
||||
* `computeMigrationEnabled: false`.
|
||||
*/
|
||||
export function isOrgMigrated({
|
||||
planType,
|
||||
orgId,
|
||||
orgFeatureFlags,
|
||||
flags,
|
||||
}: MigrationDecisionInput): boolean {
|
||||
const override = orgFeatureFlags?.["computeMigrationEnabled"];
|
||||
if (override === false) return false;
|
||||
if (override === true) return true;
|
||||
|
||||
if (!(flags?.computeMigrationEnabled ?? false)) return false;
|
||||
|
||||
const pct =
|
||||
planType === "free"
|
||||
? (flags?.computeMigrationFreePercentage ?? 0)
|
||||
: planType === "paid"
|
||||
? (flags?.computeMigrationPaidPercentage ?? 0)
|
||||
: 0; // enterprise / undefined
|
||||
|
||||
return hashBucket(orgId) < pct;
|
||||
}
|
||||
|
||||
type ResolveInput = MigrationDecisionInput & {
|
||||
baseWorkerQueue: string | undefined;
|
||||
baseEnableFastPath: boolean;
|
||||
region: string | undefined; // geo of the base queue (same whether migrated or not)
|
||||
backing: { workerQueue: string; enableFastPath: boolean } | undefined;
|
||||
envType: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Produce the target descriptor `{ workerQueue, region, enableFastPath }` for a
|
||||
* run. When the org is migrated and the region has a compute backing, the queue
|
||||
* and fast-path setting come from the MICROVM backing group; `region` is the geo
|
||||
* either way. Same-geo swap (us-east-1 -> us-east-1-next): any explicit placement
|
||||
* is a geography preference, honored by staying in-region. Applied after region
|
||||
* resolution, mirroring the scheduled-split.
|
||||
*/
|
||||
export function resolveComputeMigration({
|
||||
baseWorkerQueue,
|
||||
baseEnableFastPath,
|
||||
region,
|
||||
backing,
|
||||
envType,
|
||||
...decision
|
||||
}: ResolveInput): {
|
||||
workerQueue: string | undefined;
|
||||
region: string | undefined;
|
||||
enableFastPath: boolean;
|
||||
} {
|
||||
const passthrough = { workerQueue: baseWorkerQueue, region, enableFastPath: baseEnableFastPath };
|
||||
if (baseWorkerQueue === undefined) return passthrough;
|
||||
if (envType === "DEVELOPMENT") return passthrough;
|
||||
if (!isOrgMigrated(decision)) return passthrough;
|
||||
if (!backing) return passthrough;
|
||||
return { workerQueue: backing.workerQueue, region, enableFastPath: backing.enableFastPath };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { getMeter } from "@internal/tracing";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
baseWorkerQueue,
|
||||
matchesDisabledWorkerQueue,
|
||||
parseDisabledWorkerQueues,
|
||||
} from "./workerQueueSplit.server";
|
||||
|
||||
const meter = getMeter("run-engine-dequeue-gate");
|
||||
|
||||
const blockedDequeueCounter = meter.createCounter("run_engine.dequeue.blocked", {
|
||||
description:
|
||||
"Count of worker dequeue requests refused because the worker queue is gated off via RUN_ENGINE_DEQUEUE_DISABLED_WORKER_QUEUES",
|
||||
});
|
||||
|
||||
const disabledWorkerQueues = parseDisabledWorkerQueues(
|
||||
env.RUN_ENGINE_DEQUEUE_DISABLED_WORKER_QUEUES
|
||||
);
|
||||
|
||||
export function isWorkerQueueDequeueDisabled(workerQueue: string): boolean {
|
||||
return matchesDisabledWorkerQueue(workerQueue, disabledWorkerQueues);
|
||||
}
|
||||
|
||||
export function recordBlockedDequeue(workerQueue: string): void {
|
||||
blockedDequeueCounter.add(1, {
|
||||
worker_queue: workerQueue,
|
||||
region: baseWorkerQueue(workerQueue),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
import type { RunEngine } from "~/v3/runEngine.server";
|
||||
import { shouldIdempotencyKeyBeCleared } from "~/v3/taskStatus";
|
||||
import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
|
||||
import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server";
|
||||
import { claimOrAwait } from "~/v3/mollifier/idempotencyClaim.server";
|
||||
import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server";
|
||||
import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveIdempotencyDedupClient } from "./idempotencyResidency.server";
|
||||
import type { TraceEventConcern, TriggerTaskRequest } from "../types";
|
||||
|
||||
// In-memory per-org mollifier-enabled check, shared with `evaluateGate`
|
||||
// (same `Organization.featureFlags` JSON, no DB read). Used to gate the
|
||||
// pre-gate claim's Redis round-trip so non-mollifier orgs don't pay it
|
||||
// during staged rollout — see the comment above the claim block in
|
||||
// handleTriggerRequest.
|
||||
const resolveOrgMollifierFlag = makeResolveMollifierFlag();
|
||||
|
||||
// Claim ownership context returned to the caller when the
|
||||
// IdempotencyKeyConcern won a pre-gate claim. Caller MUST publish the
|
||||
// winning runId on pipeline success (`publishClaim`) or release the
|
||||
// claim on failure (`releaseClaim`).
|
||||
export type ClaimedIdempotency = {
|
||||
envId: string;
|
||||
taskIdentifier: string;
|
||||
idempotencyKey: string;
|
||||
// Ownership token from `claimOrAwait`. The caller's trigger pipeline
|
||||
// MUST thread this into publishClaim/releaseClaim so the buffer's
|
||||
// compare-and-act protects the slot against a stale predecessor.
|
||||
token: string;
|
||||
};
|
||||
|
||||
export type IdempotencyKeyConcernResult =
|
||||
| { isCached: true; run: TaskRun }
|
||||
| {
|
||||
isCached: false;
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
// Set when this trigger holds a pre-gate claim. The caller's
|
||||
// trigger pipeline MUST resolve the claim by either publishing
|
||||
// the runId on success or releasing on failure. Undefined when
|
||||
// the request has no idempotency key, when the buffer is
|
||||
// unavailable, or when the request is a triggerAndWait (claim
|
||||
// path skipped per plan doc).
|
||||
claim?: ClaimedIdempotency;
|
||||
};
|
||||
|
||||
export class IdempotencyKeyConcern {
|
||||
constructor(
|
||||
private readonly prisma: PrismaClientOrTransaction,
|
||||
private readonly engine: RunEngine,
|
||||
private readonly traceEventConcern: TraceEventConcern
|
||||
) {}
|
||||
|
||||
// Buffer-side idempotency dedup. Resolves an idempotency key against the
|
||||
// mollifier buffer when PG missed. Returns a SyntheticRun cast to
|
||||
// TaskRun so the route handler (which only reads run.id / run.friendlyId)
|
||||
// can echo the buffered run's friendlyId as a cached hit. Returns null
|
||||
// for any failure or miss — buffer outages must not 500 the trigger
|
||||
// hot path; we fail open to "no cache hit" and let the request through.
|
||||
private async findBufferedRunWithIdempotency(
|
||||
environmentId: string,
|
||||
organizationId: string,
|
||||
taskIdentifier: string,
|
||||
idempotencyKey: string
|
||||
): Promise<TaskRun | null> {
|
||||
const buffer = getMollifierBuffer();
|
||||
if (!buffer) return null;
|
||||
|
||||
let bufferedRunId: string | null;
|
||||
try {
|
||||
bufferedRunId = await buffer.lookupIdempotency({
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
idempotencyKey,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error("IdempotencyKeyConcern: buffer lookupIdempotency failed", {
|
||||
environmentId,
|
||||
taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
if (!bufferedRunId) return null;
|
||||
|
||||
const synthetic = await findRunByIdWithMollifierFallback({
|
||||
runId: bufferedRunId,
|
||||
environmentId,
|
||||
organizationId,
|
||||
});
|
||||
if (!synthetic) return null;
|
||||
// PG-resident path enforces idempotency-key expiry below
|
||||
// (`existingRun.idempotencyKeyExpiresAt < new Date()` clears the key
|
||||
// and lets a new run go through). The buffer path needs the same
|
||||
// check — without it a customer who passes `idempotencyKeyTTL: "2s"`
|
||||
// gets the cached buffered runId returned indefinitely, because the
|
||||
// buffer entry persists for its own (hours-long) TTL independent of
|
||||
// the customer's key TTL.
|
||||
//
|
||||
// Returning null isn't enough on its own: the trigger pipeline then
|
||||
// proceeds to `mollifyTrigger`, whose `buffer.accept` Lua dedupes by
|
||||
// `(envId, taskIdentifier, idempotencyKey)` via SETNX on the same
|
||||
// `mollifier:idempotency:*` key and would echo the stale runId as
|
||||
// `duplicate_idempotency`. Clear the buffer-side idempotency
|
||||
// binding (both the lookup and any in-flight claim) so the next
|
||||
// accept goes through as a fresh trigger. Mirrors what
|
||||
// `ResetIdempotencyKeyService` does for the explicit
|
||||
// reset-via-API path.
|
||||
if (synthetic.idempotencyKeyExpiresAt && synthetic.idempotencyKeyExpiresAt < new Date()) {
|
||||
const buffer = getMollifierBuffer();
|
||||
if (buffer) {
|
||||
try {
|
||||
await buffer.resetIdempotency({
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
idempotencyKey,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn("IdempotencyKeyConcern: failed to reset expired buffer idempotency", {
|
||||
envId: environmentId,
|
||||
taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return synthetic as unknown as TaskRun;
|
||||
}
|
||||
|
||||
async handleTriggerRequest(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined
|
||||
): Promise<IdempotencyKeyConcernResult> {
|
||||
const idempotencyKey = request.options?.idempotencyKey ?? request.body.options?.idempotencyKey;
|
||||
const idempotencyKeyExpiresAt =
|
||||
request.options?.idempotencyKeyExpiresAt ??
|
||||
resolveIdempotencyKeyTTL(request.body.options?.idempotencyKeyTTL) ??
|
||||
new Date(Date.now() + 24 * 60 * 60 * 1000 * 30); // 30 days
|
||||
|
||||
if (!idempotencyKey) {
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// Probe and clears must hit the DB where the would-be run will physically live.
|
||||
const dedupClient = await resolveIdempotencyDedupClient(
|
||||
{
|
||||
environmentForMint: {
|
||||
organizationId: request.environment.organizationId,
|
||||
id: request.environment.id,
|
||||
orgFeatureFlags: request.environment.organization?.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: request.body.options?.parentRunId,
|
||||
},
|
||||
{
|
||||
isSplitEnabled,
|
||||
fallbackClient: this.prisma,
|
||||
newClient: runOpsNewPrisma,
|
||||
legacyClient: runOpsLegacyPrisma,
|
||||
resolveMintKind: resolveRunIdMintKind,
|
||||
// `isMigrated` is intentionally omitted: until a child of a swept
|
||||
// legacy-id parent can be born on the new DB, the swept-marker override
|
||||
// would never change the answer, so a child routes by parent id-shape.
|
||||
}
|
||||
);
|
||||
|
||||
const existingRun = idempotencyKey
|
||||
? await runStore.findRun(
|
||||
{
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: request.taskId,
|
||||
},
|
||||
{
|
||||
include: {
|
||||
associatedWaitpoint: true,
|
||||
},
|
||||
},
|
||||
dedupClient
|
||||
)
|
||||
: undefined;
|
||||
|
||||
// Buffer fallback per the mollifier-idempotency design. PG missed —
|
||||
// the same key may belong to a buffered run that hasn't materialised
|
||||
// yet. Skipped when `resumeParentOnCompletion` is set: blocking a
|
||||
// parent on a buffered child via waitpoint requires a PG row that
|
||||
// doesn't exist yet. The follow-up accept's SETNX in mollifyTrigger
|
||||
// still dedupes the trigger itself; the waitpoint just doesn't fire
|
||||
// for this rare race window.
|
||||
if (!existingRun && idempotencyKey && !request.body.options?.resumeParentOnCompletion) {
|
||||
const buffered = await this.findBufferedRunWithIdempotency(
|
||||
request.environment.id,
|
||||
request.environment.organizationId,
|
||||
request.taskId,
|
||||
idempotencyKey
|
||||
);
|
||||
if (buffered) {
|
||||
return { isCached: true, run: buffered };
|
||||
}
|
||||
}
|
||||
|
||||
if (existingRun) {
|
||||
// The idempotency key has expired
|
||||
if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) {
|
||||
logger.debug("[TriggerTaskService][call] Idempotency key has expired", {
|
||||
idempotencyKey: request.options?.idempotencyKey,
|
||||
run: existingRun,
|
||||
});
|
||||
|
||||
// Update the existing run to remove the idempotency key
|
||||
await runStore.clearIdempotencyKey(
|
||||
{ byId: { runId: existingRun.id, idempotencyKey } },
|
||||
dedupClient
|
||||
);
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// If the existing run failed or was expired, we clear the key and do a new run
|
||||
if (shouldIdempotencyKeyBeCleared(existingRun.status)) {
|
||||
logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", {
|
||||
idempotencyKey: request.options?.idempotencyKey,
|
||||
runStatus: existingRun.status,
|
||||
runId: existingRun.id,
|
||||
});
|
||||
|
||||
// Update the existing run to remove the idempotency key
|
||||
await runStore.clearIdempotencyKey(
|
||||
{ byId: { runId: existingRun.id, idempotencyKey } },
|
||||
dedupClient
|
||||
);
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
|
||||
// We have an idempotent run, so we return it
|
||||
const parentRunId = request.body.options?.parentRunId;
|
||||
const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion;
|
||||
|
||||
//We're using `andWait` so we need to block the parent run with a waitpoint
|
||||
if (resumeParentOnCompletion && parentRunId) {
|
||||
// `parentRunId` comes from the request body and isn't re-validated
|
||||
// here, so confirm the parent run is in the caller's environment
|
||||
// before wiring a waitpoint against it.
|
||||
const parentRunInternalId = RunId.fromFriendlyId(parentRunId);
|
||||
const parentRunInCallerEnv = await runStore.findRun(
|
||||
{
|
||||
id: parentRunInternalId,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
},
|
||||
{ select: { id: true } },
|
||||
this.prisma
|
||||
);
|
||||
if (!parentRunInCallerEnv) {
|
||||
throw new ServiceValidationError("Parent run not found in the calling environment", 404);
|
||||
}
|
||||
|
||||
// Get or create waitpoint lazily (existing run may not have one if it was standalone)
|
||||
let associatedWaitpoint = existingRun.associatedWaitpoint;
|
||||
if (!associatedWaitpoint) {
|
||||
associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({
|
||||
runId: existingRun.id,
|
||||
projectId: request.environment.projectId,
|
||||
environmentId: request.environment.id,
|
||||
});
|
||||
}
|
||||
|
||||
await this.traceEventConcern.traceIdempotentRun(
|
||||
request,
|
||||
parentStore,
|
||||
{
|
||||
existingRun,
|
||||
idempotencyKey,
|
||||
incomplete: associatedWaitpoint.status === "PENDING",
|
||||
isError: associatedWaitpoint.outputIsError,
|
||||
},
|
||||
async (event) => {
|
||||
const spanId =
|
||||
request.options?.parentAsLinkType === "replay"
|
||||
? event.spanId
|
||||
: event.traceparent?.spanId
|
||||
? `${event.traceparent.spanId}:${event.spanId}`
|
||||
: event.spanId;
|
||||
|
||||
await this.engine.blockRunWithWaitpoint({
|
||||
runId: parentRunInternalId,
|
||||
waitpoints: associatedWaitpoint!.id,
|
||||
spanIdToComplete: spanId,
|
||||
batch: request.options?.batchId
|
||||
? {
|
||||
id: request.options.batchId,
|
||||
index: request.options.batchIndex ?? 0,
|
||||
}
|
||||
: undefined,
|
||||
projectId: request.environment.projectId,
|
||||
organizationId: request.environment.organizationId,
|
||||
tx: dedupClient,
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return { isCached: true, run: existingRun };
|
||||
}
|
||||
|
||||
// Pre-gate claim — closes the PG+buffer race during gate transition.
|
||||
// All same-key triggers serialise here before evaluateGate decides
|
||||
// PG-pass-through vs mollify. Skipped for triggerAndWait
|
||||
// (resumeParentOnCompletion) — that path bypasses the gate entirely
|
||||
// and its existing PG-side dedup is sufficient.
|
||||
//
|
||||
// Gated on the same per-org mollifier flag the gate uses, and the same
|
||||
// bypass list (debounce + oneTimeUseToken): if the gate would never mollify
|
||||
// the request, there's no buffer to serialise against and PG's unique
|
||||
// constraint already deduplicates concurrent same-key races. Skipping the
|
||||
// claim's Redis SETNX keeps its RTT off the hot path for those requests
|
||||
// during staged rollout. The org-flag check is a pure in-memory read of
|
||||
// `Organization.featureFlags`, no DB query.
|
||||
const claimEligible =
|
||||
!request.body.options?.resumeParentOnCompletion &&
|
||||
!request.body.options?.debounce &&
|
||||
!request.options?.oneTimeUseToken &&
|
||||
(await resolveOrgMollifierFlag({
|
||||
envId: request.environment.id,
|
||||
orgId: request.environment.organizationId,
|
||||
taskId: request.taskId,
|
||||
orgFeatureFlags:
|
||||
(request.environment.organization?.featureFlags as
|
||||
| Record<string, unknown>
|
||||
| null
|
||||
| undefined) ?? null,
|
||||
}));
|
||||
if (claimEligible) {
|
||||
const ttlSeconds = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS,
|
||||
Math.ceil((idempotencyKeyExpiresAt.getTime() - Date.now()) / 1000)
|
||||
)
|
||||
);
|
||||
const outcome = await claimOrAwait({
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
idempotencyKey,
|
||||
ttlSeconds,
|
||||
safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS,
|
||||
pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS,
|
||||
});
|
||||
if (outcome.kind === "resolved") {
|
||||
// Another concurrent trigger committed first. Re-resolve via the
|
||||
// existing checks: writer-side PG findFirst first (defeats
|
||||
// replica lag), then buffer fallback for the buffered case.
|
||||
const writerRun = await runStore.findRun(
|
||||
{
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: request.taskId,
|
||||
},
|
||||
{ include: { associatedWaitpoint: true } },
|
||||
dedupClient
|
||||
);
|
||||
if (writerRun) {
|
||||
return { isCached: true, run: writerRun };
|
||||
}
|
||||
const buffered = await this.findBufferedRunWithIdempotency(
|
||||
request.environment.id,
|
||||
request.environment.organizationId,
|
||||
request.taskId,
|
||||
idempotencyKey
|
||||
);
|
||||
if (buffered) {
|
||||
return { isCached: true, run: buffered };
|
||||
}
|
||||
// Claim resolved to a runId nothing can find — the run was genuinely
|
||||
// lost (claimant errored after publish, or both the PG row and buffer
|
||||
// entry TTL'd out). Terminal, not transient, so falling through to a
|
||||
// fresh trigger is the correct recovery.
|
||||
//
|
||||
// Falling through claimless doesn't duplicate runs: concurrent
|
||||
// fall-throughs converge on one run via the same dedup backstops the
|
||||
// claim layer relies on — PG's unique constraint on the idempotency key
|
||||
// (pass-through path) and `accept`'s SETNX (mollify path). Once the
|
||||
// first commits, later callers find it via the writer-PG / buffer
|
||||
// lookups above despite the stale `resolved:` slot (cleared by its ~30s
|
||||
// TTL). Residual cost is a few deduped trigger attempts, not dup runs.
|
||||
logger.warn("idempotency claim resolved but runId not findable", {
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
claimedRunId: outcome.runId,
|
||||
});
|
||||
}
|
||||
if (outcome.kind === "timed_out") {
|
||||
throw new ServiceValidationError("Idempotency claim resolution timed out", 503);
|
||||
}
|
||||
if (outcome.kind === "claimed") {
|
||||
// Caller MUST publish/release. Signalled via the result's
|
||||
// `claim` field, including the ownership token so the buffer
|
||||
// can compare-and-act on the slot we now own.
|
||||
return {
|
||||
isCached: false,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
claim: {
|
||||
envId: request.environment.id,
|
||||
taskIdentifier: request.taskId,
|
||||
idempotencyKey,
|
||||
token: outcome.token,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
resolveIdempotencyDedupClient,
|
||||
type ResolveIdempotencyClientDeps,
|
||||
} from "./idempotencyResidency.server";
|
||||
|
||||
// Distinct sentinel objects so we can assert WHICH client was selected by reference.
|
||||
const FALLBACK = { __tag: "fallback" } as never;
|
||||
const NEW_CLIENT = { __tag: "new" } as never;
|
||||
const LEGACY_CLIENT = { __tag: "legacy" } as never;
|
||||
|
||||
function makeDeps(over: Partial<ResolveIdempotencyClientDeps>): ResolveIdempotencyClientDeps {
|
||||
return {
|
||||
isSplitEnabled: async () => true,
|
||||
fallbackClient: FALLBACK,
|
||||
newClient: NEW_CLIENT,
|
||||
legacyClient: LEGACY_CLIENT,
|
||||
resolveMintKind: async () => "runOpsId",
|
||||
classify: (id) => {
|
||||
if (id.length === 26 && id[25] === "1") return "NEW";
|
||||
if (id.length === 25) return "LEGACY";
|
||||
throw new Error(`unclassifiable: ${id.length}`);
|
||||
},
|
||||
isMigrated: undefined,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const env = { organizationId: "org_1", id: "env_1", orgFeatureFlags: {} };
|
||||
|
||||
describe("resolveIdempotencyDedupClient", () => {
|
||||
it("returns the fallback client unchanged when split is disabled", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ isSplitEnabled: async () => false })
|
||||
);
|
||||
expect(client).toBe(FALLBACK);
|
||||
});
|
||||
|
||||
it("routes a root run to the NEW client when the env mints run-ops ids", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ resolveMintKind: async () => "runOpsId" })
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a root run to the LEGACY client when the env mints cuid", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: undefined },
|
||||
makeDeps({ resolveMintKind: async () => "cuid" })
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a child to the NEW client when the run-ops parent is NEW-resident", async () => {
|
||||
const runOpsParent = RunId.toFriendlyId("a".repeat(24) + "01");
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: runOpsParent },
|
||||
makeDeps({ resolveMintKind: async () => "cuid" }) // mint flag must NOT win for a child
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a child to the LEGACY client when the cuid parent is LEGACY-resident", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("b".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ resolveMintKind: async () => "runOpsId" }) // mint flag must NOT win for a child
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a swept (migrated) cuid-parent child to the NEW client", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("c".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ isMigrated: async () => true })
|
||||
);
|
||||
expect(client).toBe(NEW_CLIENT);
|
||||
});
|
||||
|
||||
it("routes a non-migrated cuid-parent child to the LEGACY client even when isMigrated is provided", async () => {
|
||||
const cuidParent = RunId.toFriendlyId("d".repeat(25));
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: cuidParent },
|
||||
makeDeps({ isMigrated: async () => false })
|
||||
);
|
||||
expect(client).toBe(LEGACY_CLIENT);
|
||||
});
|
||||
|
||||
it("falls back to the fallback client when a present parent id is unclassifiable", async () => {
|
||||
const client = await resolveIdempotencyDedupClient(
|
||||
{ environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" },
|
||||
makeDeps({})
|
||||
);
|
||||
expect(client).toBe(FALLBACK);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { ownerEngine, RunId, type Residency } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
|
||||
type MintKind = "cuid" | "runOpsId";
|
||||
|
||||
export type ResolveIdempotencyClientDeps = {
|
||||
isSplitEnabled: () => Promise<boolean>;
|
||||
fallbackClient: PrismaClientOrTransaction;
|
||||
newClient: PrismaClientOrTransaction;
|
||||
legacyClient: PrismaClientOrTransaction;
|
||||
resolveMintKind: (environment: {
|
||||
organizationId: string;
|
||||
id: string;
|
||||
orgFeatureFlags?: unknown;
|
||||
}) => Promise<MintKind>;
|
||||
classify?: (id: string) => Residency;
|
||||
isMigrated?: (id: string) => Promise<boolean>;
|
||||
};
|
||||
|
||||
export async function resolveIdempotencyDedupClient(
|
||||
args: {
|
||||
environmentForMint: { organizationId: string; id: string; orgFeatureFlags?: unknown };
|
||||
parentRunFriendlyId: string | undefined;
|
||||
},
|
||||
deps: ResolveIdempotencyClientDeps
|
||||
): Promise<PrismaClientOrTransaction> {
|
||||
if (!(await deps.isSplitEnabled())) {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
|
||||
const classify = deps.classify ?? ownerEngine;
|
||||
const clientFor = (residency: Residency): PrismaClientOrTransaction =>
|
||||
residency === "NEW" ? deps.newClient : deps.legacyClient;
|
||||
|
||||
if (args.parentRunFriendlyId) {
|
||||
let parentInternalId: string;
|
||||
try {
|
||||
parentInternalId = RunId.fromFriendlyId(args.parentRunFriendlyId);
|
||||
} catch {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
let residency: Residency;
|
||||
try {
|
||||
residency = classify(parentInternalId);
|
||||
} catch {
|
||||
return deps.fallbackClient;
|
||||
}
|
||||
if (residency === "LEGACY" && deps.isMigrated && (await deps.isMigrated(parentInternalId))) {
|
||||
return deps.newClient;
|
||||
}
|
||||
return clientFor(residency);
|
||||
}
|
||||
|
||||
const kind = await deps.resolveMintKind(args.environmentForMint);
|
||||
return clientFor(kind === "runOpsId" ? "NEW" : "LEGACY");
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { IOPacket } from "@trigger.dev/core/v3";
|
||||
import { packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import type { PayloadProcessor, TriggerTaskRequest } from "../types";
|
||||
import { env } from "~/env.server";
|
||||
import { startActiveSpan } from "~/v3/tracer.server";
|
||||
import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
export class DefaultPayloadProcessor implements PayloadProcessor {
|
||||
async process(request: TriggerTaskRequest): Promise<IOPacket> {
|
||||
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
||||
const payload = request.body.payload;
|
||||
const payloadType = request.body.options?.payloadType ?? "application/json";
|
||||
|
||||
const packet = this.#createPayloadPacket(payload, payloadType);
|
||||
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading, size } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
span.setAttribute("needsOffloading", needsOffloading);
|
||||
// When the caller already offloaded the payload (payloadType "application/store"), the
|
||||
// packet here is just the small object-store reference, so `size` measures the reference,
|
||||
// not the payload. Prefer the caller-reported pre-offload size when it's provided so the
|
||||
// span reflects the real payload size. For inline payloads the two agree.
|
||||
span.setAttribute("size", request.body.options?.payloadSize ?? size);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${request.friendlyId}/payload.json`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
request.environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
throw new ServiceValidationError("Failed to upload large payload to object store", 500); // This is retryable
|
||||
}
|
||||
|
||||
return {
|
||||
data: uploadedFilename!,
|
||||
dataType: "application/store",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
#createPayloadPacket(payload: any, payloadType: string): IOPacket {
|
||||
if (payloadType === "application/json") {
|
||||
return { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
}
|
||||
|
||||
if (typeof payload === "string") {
|
||||
return { data: payload, dataType: payloadType };
|
||||
}
|
||||
|
||||
return { dataType: payloadType };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
import { sanitizeQueueName } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
|
||||
import type {
|
||||
LockedBackgroundWorker,
|
||||
QueueManager,
|
||||
QueueProperties,
|
||||
QueueValidationResult,
|
||||
TriggerTaskRequest,
|
||||
} from "../types";
|
||||
import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server";
|
||||
import type { RunEngine } from "~/v3/runEngine.server";
|
||||
import { env } from "~/env.server";
|
||||
import { tryCatch } from "@trigger.dev/core/v3";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
import { isInfrastructureError } from "~/utils/prismaErrors";
|
||||
import {
|
||||
createCache,
|
||||
createLRUMemoryStore,
|
||||
DefaultStatefulContext,
|
||||
Namespace,
|
||||
} from "@internal/cache";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import type { TaskMetadataCache, TaskMetadataEntry } from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import {
|
||||
recordTaskMetaResolve,
|
||||
type TaskMetaResolveSource,
|
||||
} from "~/services/taskMetadataCacheTelemetry.server";
|
||||
|
||||
// LRU cache for environment queue sizes to reduce Redis calls
|
||||
const queueSizeCache = singleton("queueSizeCache", () => {
|
||||
const ctx = new DefaultStatefulContext();
|
||||
const memory = createLRUMemoryStore(env.QUEUE_SIZE_CACHE_MAX_SIZE, "queue-size-cache");
|
||||
|
||||
return createCache({
|
||||
queueSize: new Namespace<number>(ctx, {
|
||||
stores: [memory],
|
||||
fresh: env.QUEUE_SIZE_CACHE_TTL_MS,
|
||||
stale: env.QUEUE_SIZE_CACHE_TTL_MS + 1000,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Extract the queue name from a queue option that may be:
|
||||
* - An object with a string `name` property: { name: "queue-name" }
|
||||
* - A double-wrapped object (bug case): { name: { name: "queue-name", ... } }
|
||||
*
|
||||
* This handles the case where the SDK accidentally double-wraps the queue
|
||||
* option when it's already an object with a name property.
|
||||
*/
|
||||
function extractQueueName(queue: { name?: unknown } | undefined): string | undefined {
|
||||
if (!queue?.name) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Normal case: queue.name is a string
|
||||
if (typeof queue.name === "string") {
|
||||
return queue.name;
|
||||
}
|
||||
|
||||
// Double-wrapped case: queue.name is an object with its own name property
|
||||
if (typeof queue.name === "object" && queue.name !== null && "name" in queue.name) {
|
||||
const innerName = (queue.name as { name: unknown }).name;
|
||||
if (typeof innerName === "string") {
|
||||
return innerName;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export class DefaultQueueManager implements QueueManager {
|
||||
private readonly replicaPrisma: PrismaClientOrTransaction;
|
||||
private readonly taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaClientOrTransaction,
|
||||
private readonly engine: RunEngine,
|
||||
replicaPrisma?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
this.replicaPrisma = replicaPrisma ?? prisma;
|
||||
this.taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
async resolveQueueProperties(
|
||||
request: TriggerTaskRequest,
|
||||
lockedBackgroundWorker?: LockedBackgroundWorker
|
||||
): Promise<QueueProperties> {
|
||||
let queueName: string;
|
||||
let lockedQueueId: string | undefined;
|
||||
let taskTtl: string | null | undefined;
|
||||
let taskKind: string | undefined;
|
||||
|
||||
// Determine queue name based on lockToVersion and provided options
|
||||
if (lockedBackgroundWorker) {
|
||||
// Task is locked to a specific worker version
|
||||
const specifiedQueueName = extractQueueName(request.body.options?.queue);
|
||||
|
||||
if (specifiedQueueName) {
|
||||
// A specific queue name is provided, validate it exists for the locked worker.
|
||||
// Pre-existing query — not cached because TaskQueue rows can be added or
|
||||
// removed independently of BackgroundWorkerTask, and a stale "queue exists"
|
||||
// claim would silently route to the wrong queue.
|
||||
const specifiedQueue = await this.prisma.taskQueue.findFirst({
|
||||
where: {
|
||||
name: specifiedQueueName,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
workers: { some: { id: lockedBackgroundWorker.id } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!specifiedQueue) {
|
||||
throw new ServiceValidationError(
|
||||
`Specified queue '${specifiedQueueName}' not found or not associated with locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
// Use the validated queue name directly
|
||||
queueName = specifiedQueue.name;
|
||||
lockedQueueId = specifiedQueue.id;
|
||||
|
||||
// Pull `triggerSource` (for `taskKind` annotation) and `ttl` from cache.
|
||||
// On cache hit this is 0 PG queries; on miss the helper falls back to
|
||||
// a BackgroundWorkerTask lookup and back-fills the cache.
|
||||
//
|
||||
// If the task slug isn't on this locked worker version, we tolerate
|
||||
// the missing row and fall through with `taskKind = undefined`
|
||||
// (coalesced to "STANDARD" downstream) and `taskTtl = undefined`.
|
||||
// This matches main's pre-PR behavior — the no-override branch below
|
||||
// still throws because there's no queue to route to in that case,
|
||||
// but here the caller already named the queue.
|
||||
const lockedMeta = await this.resolveLockedTaskMetadata(
|
||||
lockedBackgroundWorker.id,
|
||||
request.environment.id,
|
||||
request.taskId
|
||||
);
|
||||
|
||||
if (request.body.options?.ttl === undefined) {
|
||||
taskTtl = lockedMeta?.ttl ?? undefined;
|
||||
}
|
||||
taskKind = lockedMeta?.triggerSource;
|
||||
} else {
|
||||
// No queue override - resolve default queue + TTL + triggerSource via cache,
|
||||
// falling back to a single BackgroundWorkerTask lookup on miss.
|
||||
const lockedMeta = await this.resolveLockedTaskMetadata(
|
||||
lockedBackgroundWorker.id,
|
||||
request.environment.id,
|
||||
request.taskId
|
||||
);
|
||||
|
||||
if (!lockedMeta) {
|
||||
throw new ServiceValidationError(
|
||||
`Task '${request.taskId}' not found on locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
taskTtl = lockedMeta.ttl;
|
||||
|
||||
if (!lockedMeta.queueName) {
|
||||
// This case should ideally be prevented by earlier checks or schema constraints,
|
||||
// but handle it defensively.
|
||||
logger.error("Task found on locked version, but has no associated queue record", {
|
||||
taskId: request.taskId,
|
||||
workerId: lockedBackgroundWorker.id,
|
||||
version: lockedBackgroundWorker.version,
|
||||
});
|
||||
throw new ServiceValidationError(
|
||||
`Default queue configuration for task '${request.taskId}' missing on locked version '${
|
||||
lockedBackgroundWorker.version ?? "<unknown>"
|
||||
}'.`
|
||||
);
|
||||
}
|
||||
|
||||
// Use the task's default queue name
|
||||
queueName = lockedMeta.queueName;
|
||||
lockedQueueId = lockedMeta.queueId ?? undefined;
|
||||
taskKind = lockedMeta.triggerSource;
|
||||
}
|
||||
} else {
|
||||
// Task is not locked to a specific version, use regular logic
|
||||
if (request.body.options?.lockToVersion) {
|
||||
// This should only happen if the findFirst failed, indicating the version doesn't exist
|
||||
throw new ServiceValidationError(
|
||||
`Task locked to version '${request.body.options.lockToVersion}', but no worker found with that version.`
|
||||
);
|
||||
}
|
||||
|
||||
// Get queue name using the helper for non-locked case (handles provided name or finds default)
|
||||
const taskInfo = await this.getTaskQueueInfo(request);
|
||||
queueName = taskInfo.queueName;
|
||||
taskTtl = taskInfo.taskTtl;
|
||||
taskKind = taskInfo.taskKind;
|
||||
}
|
||||
|
||||
// Sanitize the final determined queue name once
|
||||
const sanitizedQueueName = sanitizeQueueName(queueName);
|
||||
|
||||
// Check that the queuename is not an empty string
|
||||
if (!sanitizedQueueName) {
|
||||
queueName = sanitizeQueueName(`task/${request.taskId}`); // Fallback if sanitization results in empty
|
||||
} else {
|
||||
queueName = sanitizedQueueName;
|
||||
}
|
||||
|
||||
return {
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
taskTtl,
|
||||
taskKind,
|
||||
};
|
||||
}
|
||||
|
||||
private async getTaskQueueInfo(
|
||||
request: TriggerTaskRequest
|
||||
): Promise<{ queueName: string; taskTtl?: string | null; taskKind?: string | undefined }> {
|
||||
const { taskId, environment, body } = request;
|
||||
const { queue } = body.options ?? {};
|
||||
|
||||
// Use extractQueueName to handle double-wrapped queue objects
|
||||
const overriddenQueueName = extractQueueName(queue);
|
||||
|
||||
const defaultQueueName = `task/${taskId}`;
|
||||
|
||||
// Resolve the current worker's task metadata via cache (HGET on warm path,
|
||||
// BackgroundWorkerTask findFirst + cache back-fill on miss). When this hits,
|
||||
// both the queue-override + TTL caller and the default-queue caller satisfy
|
||||
// their full result without any database query.
|
||||
const meta = await this.resolveCurrentTaskMetadata(environment, taskId);
|
||||
|
||||
if (overriddenQueueName) {
|
||||
// Caller already named the queue. We only need triggerSource (for taskKind)
|
||||
// and ttl (for the call site to coalesce against body.options.ttl).
|
||||
return {
|
||||
queueName: overriddenQueueName,
|
||||
taskTtl: meta?.ttl ?? undefined,
|
||||
taskKind: meta?.triggerSource,
|
||||
};
|
||||
}
|
||||
|
||||
if (!meta) {
|
||||
logger.debug("Failed to get queue name: No worker or task found", {
|
||||
taskId,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
return { queueName: defaultQueueName, taskTtl: undefined };
|
||||
}
|
||||
|
||||
if (!meta.queueName) {
|
||||
logger.debug("Failed to get queue name: No queue found", {
|
||||
taskId,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
return { queueName: defaultQueueName, taskTtl: meta.ttl, taskKind: meta.triggerSource };
|
||||
}
|
||||
|
||||
return { queueName: meta.queueName, taskTtl: meta.ttl, taskKind: meta.triggerSource };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve task metadata for a locked-version trigger. Reads from the
|
||||
* `task-meta:by-worker:{workerId}` Redis hash; falls back to a single
|
||||
* BackgroundWorkerTask findFirst on miss and back-fills the cache.
|
||||
*
|
||||
* Returns null when no BackgroundWorkerTask row exists.
|
||||
*/
|
||||
private async resolveLockedTaskMetadata(
|
||||
workerId: string,
|
||||
environmentId: string,
|
||||
slug: string
|
||||
): Promise<TaskMetadataEntry | null> {
|
||||
const cached = await this.taskMetaCache.getByWorker(workerId, slug);
|
||||
if (cached) {
|
||||
recordTaskMetaResolve("locked", "cache");
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Cache miss. Read the row from the replica first; if the replica comes
|
||||
// back empty, re-check the writer before concluding the task is missing.
|
||||
// The locked worker itself was just resolved on the writer (see
|
||||
// triggerTask.server.ts), so a replica that returns no row here is stale,
|
||||
// not authoritative. Trusting a stale-replica negative throws a
|
||||
// non-retryable "not found on locked version" for a task that is in fact
|
||||
// registered. The writer read only runs on this rare miss-then-empty path,
|
||||
// never on the hot path.
|
||||
let row = await this.findLockedTaskRow(this.replicaPrisma, workerId, environmentId, slug);
|
||||
let source: TaskMetaResolveSource = "replica";
|
||||
|
||||
if (!row && this.replicaPrisma !== this.prisma) {
|
||||
row = await this.findLockedTaskRow(this.prisma, workerId, environmentId, slug);
|
||||
|
||||
if (row) {
|
||||
source = "writer";
|
||||
logger.warn("Locked task metadata missing on replica but found on writer", {
|
||||
workerId,
|
||||
environmentId,
|
||||
slug,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!row) {
|
||||
recordTaskMetaResolve("locked", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
recordTaskMetaResolve("locked", source);
|
||||
|
||||
const entry: TaskMetadataEntry = {
|
||||
slug,
|
||||
ttl: row.ttl,
|
||||
triggerSource: row.triggerSource,
|
||||
queueId: row.queue?.id ?? null,
|
||||
queueName: row.queue?.name ?? "",
|
||||
};
|
||||
|
||||
// Fire-and-forget back-fill — `setByWorker` upserts the single field and
|
||||
// refreshes the hash TTL. Errors are logged inside the cache and swallowed.
|
||||
void this.taskMetaCache.setByWorker(workerId, entry);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
private findLockedTaskRow(
|
||||
client: PrismaClientOrTransaction,
|
||||
workerId: string,
|
||||
environmentId: string,
|
||||
slug: string
|
||||
) {
|
||||
return client.backgroundWorkerTask.findFirst({
|
||||
where: { workerId, runtimeEnvironmentId: environmentId, slug },
|
||||
select: {
|
||||
ttl: true,
|
||||
triggerSource: true,
|
||||
queue: { select: { id: true, name: true } },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve task metadata for a non-locked trigger. Reads from the
|
||||
* `task-meta:env:{envId}` Redis hash; falls back to
|
||||
* findCurrentWorkerFromEnvironment + a single BackgroundWorkerTask findFirst
|
||||
* on miss and back-fills both keyspaces.
|
||||
*
|
||||
* Returns null when no current worker or task can be resolved.
|
||||
*/
|
||||
private async resolveCurrentTaskMetadata(
|
||||
environment: AuthenticatedEnvironment,
|
||||
slug: string
|
||||
): Promise<TaskMetadataEntry | null> {
|
||||
const cached = await this.taskMetaCache.getCurrent(environment.id, slug);
|
||||
if (cached) {
|
||||
recordTaskMetaResolve("current", "cache");
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Cold cache: discover the current worker for the env. Replica is fine —
|
||||
// the adjacent BackgroundWorkerTask lookup below uses `replicaPrisma` too
|
||||
// (replica lag for "just deployed" is bounded the same way for both
|
||||
// queries; reading from the writer here would only widen the window).
|
||||
const worker = await findCurrentWorkerFromEnvironment(environment, this.replicaPrisma);
|
||||
if (!worker) {
|
||||
recordTaskMetaResolve("current", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = await this.replicaPrisma.backgroundWorkerTask.findFirst({
|
||||
where: { workerId: worker.id, runtimeEnvironmentId: environment.id, slug },
|
||||
select: {
|
||||
ttl: true,
|
||||
triggerSource: true,
|
||||
queue: { select: { id: true, name: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!row) {
|
||||
recordTaskMetaResolve("current", "miss");
|
||||
return null;
|
||||
}
|
||||
|
||||
recordTaskMetaResolve("current", "replica");
|
||||
|
||||
const entry: TaskMetadataEntry = {
|
||||
slug,
|
||||
ttl: row.ttl,
|
||||
triggerSource: row.triggerSource,
|
||||
queueId: row.queue?.id ?? null,
|
||||
queueName: row.queue?.name ?? "",
|
||||
};
|
||||
|
||||
// Fire-and-forget back-fill — atomically upserts the slug into both
|
||||
// keyspaces so a subsequent locked-or-not trigger hits the cache. The
|
||||
// env-keyspace TTL is preserved (promotion owns it); the by-worker TTL
|
||||
// is refreshed (sliding window keeps active workers warm).
|
||||
void this.taskMetaCache.setByCurrentWorker(environment.id, worker.id, entry);
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
async validateQueueLimits(
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string,
|
||||
itemsToAdd?: number
|
||||
): Promise<QueueValidationResult> {
|
||||
const queueSizeGuard = await guardQueueSizeLimitsForQueue(
|
||||
this.engine,
|
||||
environment,
|
||||
queueName,
|
||||
itemsToAdd
|
||||
);
|
||||
|
||||
logger.debug("Queue size guard result", {
|
||||
queueSizeGuard,
|
||||
queueName,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
type: environment.type,
|
||||
organization: environment.organization,
|
||||
project: environment.project,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
ok: queueSizeGuard.isWithinLimits,
|
||||
maximumSize: queueSizeGuard.maximumSize ?? 0,
|
||||
queueSize: queueSizeGuard.queueSize ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
async getWorkerQueue(
|
||||
environment: AuthenticatedEnvironment,
|
||||
regionOverride?: string
|
||||
): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined> {
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
return { masterQueue: environment.id, enableFastPath: true };
|
||||
}
|
||||
|
||||
const workerGroupService = new WorkerGroupService({
|
||||
prisma: this.prisma,
|
||||
engine: this.engine,
|
||||
});
|
||||
|
||||
const [error, workerGroup] = await tryCatch(
|
||||
workerGroupService.getDefaultWorkerGroupForProject({
|
||||
projectId: environment.projectId,
|
||||
regionOverride,
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
// getDefaultWorkerGroupForProject queries the writer DB. A Prisma
|
||||
// infrastructure error (e.g. P1001 "Can't reach database server", whose
|
||||
// message carries the DB hostname) must NOT be promoted into a
|
||||
// client-facing ServiceValidationError: that leaks internal infra detail
|
||||
// to the API client (the SDK echoes it into the run view) and
|
||||
// mis-classifies a transient outage as a non-retryable 422. Let it
|
||||
// propagate to the route's generic 500 handler (scrubbed + retryable);
|
||||
// only wrap genuine domain failures.
|
||||
if (isInfrastructureError(error)) {
|
||||
throw error;
|
||||
}
|
||||
throw new ServiceValidationError(error.message);
|
||||
}
|
||||
|
||||
if (!workerGroup) {
|
||||
throw new ServiceValidationError("No worker group found");
|
||||
}
|
||||
|
||||
return {
|
||||
masterQueue: workerGroup.masterQueue,
|
||||
enableFastPath: workerGroup.enableFastPath,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function getMaximumSizeForEnvironment(
|
||||
environment: AuthenticatedEnvironment
|
||||
): number | undefined {
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
return environment.organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE;
|
||||
} else {
|
||||
return environment.organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
async function guardQueueSizeLimitsForQueue(
|
||||
engine: RunEngine,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string,
|
||||
itemsToAdd: number = 1
|
||||
) {
|
||||
const maximumSize = getMaximumSizeForEnvironment(environment);
|
||||
|
||||
if (typeof maximumSize === "undefined") {
|
||||
return { isWithinLimits: true };
|
||||
}
|
||||
|
||||
const queueSize = await getCachedQueueSize(engine, environment, queueName);
|
||||
const projectedSize = queueSize + itemsToAdd;
|
||||
|
||||
return {
|
||||
isWithinLimits: projectedSize <= maximumSize,
|
||||
maximumSize,
|
||||
queueSize,
|
||||
};
|
||||
}
|
||||
|
||||
async function getCachedQueueSize(
|
||||
engine: RunEngine,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueName: string
|
||||
): Promise<number> {
|
||||
if (!env.QUEUE_SIZE_CACHE_ENABLED) {
|
||||
return engine.lengthOfQueue(environment, queueName);
|
||||
}
|
||||
|
||||
const cacheKey = `${environment.id}:${queueName}`;
|
||||
const result = await queueSizeCache.queueSize.swr(cacheKey, async () => {
|
||||
return engine.lengthOfQueue(environment, queueName);
|
||||
});
|
||||
|
||||
return result.val ?? 0;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import {
|
||||
$replica as defaultLegacyReplica,
|
||||
runOpsNewPrisma as defaultNewPrimary,
|
||||
runOpsNewReplica as defaultNewClient,
|
||||
runOpsSplitReadEnabled as defaultSplitReadEnabled,
|
||||
} from "~/db.server";
|
||||
import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server";
|
||||
|
||||
type ResolveWaitpointDeps = {
|
||||
newClient?: PrismaReplicaClient;
|
||||
legacyReplica?: PrismaReplicaClient;
|
||||
newPrimary?: PrismaReplicaClient;
|
||||
splitEnabled?: boolean;
|
||||
isPastRetention?: (id: string) => boolean;
|
||||
};
|
||||
|
||||
// Safe defaults matching the deps `complete`/`callback` pass, so a bare caller still fans
|
||||
// out to the dedicated run-ops replica (NEW-resident waitpoints) before control-plane.
|
||||
export type ResolveWaitpointReadThroughDefaults = {
|
||||
newClient: PrismaReplicaClient;
|
||||
legacyReplica: PrismaReplicaClient;
|
||||
newPrimary: PrismaReplicaClient;
|
||||
splitEnabled: boolean;
|
||||
};
|
||||
|
||||
const productionDefaults: ResolveWaitpointReadThroughDefaults = {
|
||||
newClient: defaultNewClient,
|
||||
legacyReplica: defaultLegacyReplica,
|
||||
newPrimary: defaultNewPrimary as unknown as PrismaReplicaClient,
|
||||
splitEnabled: defaultSplitReadEnabled,
|
||||
};
|
||||
|
||||
export async function resolveWaitpointThroughReadThrough<T>(opts: {
|
||||
waitpointId: string;
|
||||
environmentId: string;
|
||||
read: (client: PrismaReplicaClient) => Promise<T | null>;
|
||||
deps?: ResolveWaitpointDeps;
|
||||
defaults?: ResolveWaitpointReadThroughDefaults;
|
||||
}): Promise<T | null> {
|
||||
const defaults = opts.defaults ?? productionDefaults;
|
||||
|
||||
const splitEnabled = opts.deps?.splitEnabled ?? defaults.splitEnabled;
|
||||
|
||||
const result = await readThroughRun({
|
||||
runId: opts.waitpointId,
|
||||
environmentId: opts.environmentId,
|
||||
readNew: (client) => opts.read(client),
|
||||
readLegacy: (replica) => opts.read(replica),
|
||||
deps: {
|
||||
splitEnabled,
|
||||
newClient: opts.deps?.newClient ?? defaults.newClient,
|
||||
legacyReplica: opts.deps?.legacyReplica ?? defaults.legacyReplica,
|
||||
isPastRetention: opts.deps?.isPastRetention,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.source === "new" || result.source === "legacy-replica") {
|
||||
return result.value;
|
||||
}
|
||||
// past-retention is an intentional not-found: the token is gone.
|
||||
if (result.source === "past-retention") {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read-your-writes fallback for a token completed immediately after mint, before it replicated:
|
||||
// re-read from the run-ops PRIMARY only. We deliberately never read the control-plane/legacy
|
||||
// primary here (that is the load the replica-only read-through exists to shed), so a legacy-resident
|
||||
// token that misses its replica stays a miss and the caller retries, rather than adding primary load.
|
||||
const fromNewPrimary = await opts.read(opts.deps?.newPrimary ?? defaults.newPrimary);
|
||||
if (fromNewPrimary != null) {
|
||||
return fromNewPrimary;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { SemanticInternalAttributes } from "@trigger.dev/core/v3/semanticInternalAttributes";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import type { IEventRepository } from "~/v3/eventRepository/eventRepository.types";
|
||||
import { getEventRepository } from "~/v3/eventRepository/index.server";
|
||||
import type { TracedEventSpan, TraceEventConcern, TriggerTaskRequest } from "../types";
|
||||
|
||||
export class DefaultTraceEventsConcern implements TraceEventConcern {
|
||||
async #getEventRepository(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined
|
||||
): Promise<{ repository: IEventRepository; store: string }> {
|
||||
return await getEventRepository(
|
||||
request.environment.organization.id,
|
||||
request.environment.organization.featureFlags as Record<string, unknown>,
|
||||
parentStore
|
||||
);
|
||||
}
|
||||
|
||||
async traceRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
request.taskId,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {},
|
||||
style: {
|
||||
icon: request.options?.customIcon ?? "task",
|
||||
},
|
||||
},
|
||||
incomplete: true,
|
||||
immediate: true,
|
||||
startTime: request.options?.overrideCreatedAt
|
||||
? BigInt(request.options.overrideCreatedAt.getTime()) * BigInt(1000000)
|
||||
: undefined,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async traceIdempotentRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
idempotencyKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { existingRun, idempotencyKey, incomplete, isError } = options;
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
`${request.taskId} (cached)`,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {
|
||||
[SemanticInternalAttributes.ORIGINAL_RUN_ID]: existingRun.friendlyId,
|
||||
},
|
||||
style: {
|
||||
icon: "task-cached",
|
||||
},
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
incomplete,
|
||||
isError,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
//log a message
|
||||
await repository.recordEvent(
|
||||
`There's an existing run for idempotencyKey: ${idempotencyKey}`,
|
||||
{
|
||||
taskSlug: request.taskId,
|
||||
environment: request.environment,
|
||||
attributes: {
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
context: request.options?.traceContext,
|
||||
parentId: event.spanId,
|
||||
}
|
||||
);
|
||||
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async traceDebouncedRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
debounceKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
const { existingRun, debounceKey, incomplete, isError } = options;
|
||||
const { repository, store } = await this.#getEventRepository(request, parentStore);
|
||||
|
||||
return await repository.traceEvent(
|
||||
`${request.taskId} (debounced)`,
|
||||
{
|
||||
context: request.options?.traceContext,
|
||||
spanParentAsLink: request.options?.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: request.environment,
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {
|
||||
[SemanticInternalAttributes.ORIGINAL_RUN_ID]: existingRun.friendlyId,
|
||||
},
|
||||
style: {
|
||||
icon: "task-cached",
|
||||
},
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
incomplete,
|
||||
isError,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext, traceparent) => {
|
||||
// Log a message about the debounced trigger
|
||||
await repository.recordEvent(`Debounced: using existing run with key "${debounceKey}"`, {
|
||||
taskSlug: request.taskId,
|
||||
environment: request.environment,
|
||||
attributes: {
|
||||
runId: existingRun.friendlyId,
|
||||
},
|
||||
context: request.options?.traceContext,
|
||||
parentId: event.spanId,
|
||||
});
|
||||
|
||||
return await callback(
|
||||
{
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext,
|
||||
traceparent,
|
||||
setAttribute: (key, value) => event.setAttribute(key as any, value),
|
||||
failWithError: event.failWithError.bind(event),
|
||||
stop: event.stop.bind(event),
|
||||
},
|
||||
store
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/core/v3";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { env } from "~/env.server";
|
||||
import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
function packetExtensionForDataType(dataType: string): string {
|
||||
switch (dataType) {
|
||||
case "application/json":
|
||||
case "application/super+json":
|
||||
return "json";
|
||||
case "text/plain":
|
||||
return "txt";
|
||||
default:
|
||||
return "txt";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloads large waitpoint completion payloads to object store (same threshold and
|
||||
* upload path pattern as DefaultPayloadProcessor). Object key prefix should use the
|
||||
* waitpoint friendly id folder, e.g. `${WaitpointId.toFriendlyId(internalId)}/token`.
|
||||
* Replaces no-op conditionallyExportPacket usage in webapp routes where apiClientManager is unset.
|
||||
*/
|
||||
export async function processWaitpointCompletionPacket(
|
||||
packet: IOPacket,
|
||||
environment: AuthenticatedEnvironment,
|
||||
pathPrefix: string
|
||||
): Promise<IOPacket> {
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading, size: _size } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${pathPrefix}.${packetExtensionForDataType(packet.dataType)}`;
|
||||
|
||||
const [uploadError, uploadedFilename] = await tryCatch(
|
||||
uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
environment,
|
||||
env.OBJECT_STORE_DEFAULT_PROTOCOL
|
||||
)
|
||||
);
|
||||
|
||||
if (uploadError) {
|
||||
logger.error("Failed to upload large waitpoint to object store", {
|
||||
error: uploadError,
|
||||
filename,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
throw new ServiceValidationError("Failed to upload large waitpoint to object store", 500);
|
||||
}
|
||||
|
||||
return {
|
||||
data: uploadedFilename!,
|
||||
dataType: "application/store",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { WorkerQueueClass } from "@trigger.dev/core/v3/workers";
|
||||
import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags";
|
||||
|
||||
/**
|
||||
* Suffix appended to a region's worker queue name to route scheduled-lineage
|
||||
* runs onto their own Redis list (e.g. `us-nyc-3` -> `us-nyc-3:scheduled`). A
|
||||
* dedicated consumer fleet dequeues the suffixed list so the top-of-hour
|
||||
* scheduled-cron herd can't starve standard/agent run startup. The worker queue
|
||||
* name is opaque everywhere downstream (it's only ever `:`-joined into a Redis
|
||||
* key and persisted on the run), so encoding the class in the suffix needs no
|
||||
* Lua, envelope, or resolver changes.
|
||||
*/
|
||||
export const SCHEDULED_WORKER_QUEUE_SUFFIX = ":scheduled";
|
||||
|
||||
/**
|
||||
* Recover the base region a worker queue belongs to by stripping any split
|
||||
* suffix (e.g. `us-nyc-3:scheduled` -> `us-nyc-3`). Region/masterQueue names are
|
||||
* either `<name>` or `<projectId>-<name>` and never contain a colon, so the
|
||||
* region is everything before the first `:`. Use this wherever a worker queue is
|
||||
* read as a region — for display, filtering, or as a region override — so
|
||||
* scheduled-split runs group under their real region instead of a phantom one.
|
||||
* Idempotent; returns the input unchanged when there's no suffix. A nullish
|
||||
* worker queue (e.g. from a synthetic run snapshot) passes straight through.
|
||||
*/
|
||||
export function baseWorkerQueue(workerQueue: string): string;
|
||||
export function baseWorkerQueue(workerQueue: string | null | undefined): string | null | undefined;
|
||||
export function baseWorkerQueue(workerQueue: string | null | undefined): string | null | undefined {
|
||||
if (workerQueue == null) {
|
||||
return workerQueue;
|
||||
}
|
||||
|
||||
const colon = workerQueue.indexOf(":");
|
||||
return colon === -1 ? workerQueue : workerQueue.slice(0, colon);
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing region for read surfaces: the explicit geo region if set, else the
|
||||
* region derived from the worker queue, else undefined. Use everywhere a run's
|
||||
* region is displayed so an empty queue never surfaces as `""` and all surfaces
|
||||
* agree. Not for query keys — those want the raw worker queue, not this fallback.
|
||||
*/
|
||||
export function regionForDisplay(
|
||||
region: string | null | undefined,
|
||||
workerQueue: string | null | undefined
|
||||
): string | undefined {
|
||||
return region || (workerQueue ? baseWorkerQueue(workerQueue) : undefined);
|
||||
}
|
||||
|
||||
/** `TriggerSource` value used for runs originating from a schedule. */
|
||||
const SCHEDULE_TRIGGER_SOURCE = "schedule";
|
||||
|
||||
/**
|
||||
* Resolve whether the scheduled worker-queue split is enabled for a run, reading
|
||||
* only the in-memory org feature-flags JSON (already loaded on the authenticated
|
||||
* environment) — never a DB query, so it is safe on the trigger hot path.
|
||||
*
|
||||
* Precedence: a per-org override wins in BOTH directions; the global default is
|
||||
* used only when the org has not set the flag.
|
||||
*/
|
||||
export function resolveScheduledQueueSplitEnabled({
|
||||
orgFeatureFlags,
|
||||
globalDefault,
|
||||
}: {
|
||||
orgFeatureFlags: Record<string, unknown> | null | undefined;
|
||||
globalDefault: boolean;
|
||||
}): boolean {
|
||||
const override = orgFeatureFlags?.[FEATURE_FLAG.workerQueueScheduledSplitEnabled];
|
||||
|
||||
if (override !== undefined) {
|
||||
const parsed =
|
||||
FeatureFlagCatalog[FEATURE_FLAG.workerQueueScheduledSplitEnabled].safeParse(override);
|
||||
|
||||
if (parsed.success) {
|
||||
return parsed.data;
|
||||
}
|
||||
}
|
||||
|
||||
return globalDefault;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the worker queue a run should be enqueued onto. Runs in a scheduled
|
||||
* lineage (`rootTriggerSource === "schedule"`, which propagates from a scheduled
|
||||
* root down to every descendant) route to the suffixed list when the split is
|
||||
* enabled; everything else is unchanged. Idempotent — never double-suffixes.
|
||||
*/
|
||||
export function workerQueueForRun({
|
||||
workerQueue,
|
||||
rootTriggerSource,
|
||||
splitEnabled,
|
||||
}: {
|
||||
workerQueue: string;
|
||||
rootTriggerSource: string | undefined;
|
||||
splitEnabled: boolean;
|
||||
}): string {
|
||||
if (
|
||||
!splitEnabled ||
|
||||
rootTriggerSource !== SCHEDULE_TRIGGER_SOURCE ||
|
||||
workerQueue.endsWith(SCHEDULED_WORKER_QUEUE_SUFFIX)
|
||||
) {
|
||||
return workerQueue;
|
||||
}
|
||||
|
||||
return `${workerQueue}${SCHEDULED_WORKER_QUEUE_SUFFIX}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumer-side counterpart to {@link workerQueueForRun}: given a worker's base
|
||||
* (region) queue and the requested queue class, return the worker queue to
|
||||
* dequeue from. `"scheduled"` targets the suffixed list; anything else is the
|
||||
* base queue. The server always derives this from the authenticated worker's
|
||||
* own `masterQueue`, so a token can only ever reach its own region's queues.
|
||||
* Idempotent — never double-suffixes.
|
||||
*/
|
||||
export function workerQueueForClass(
|
||||
masterQueue: string,
|
||||
queueClass: WorkerQueueClass | undefined
|
||||
): string {
|
||||
if (queueClass === "scheduled" && !masterQueue.endsWith(SCHEDULED_WORKER_QUEUE_SUFFIX)) {
|
||||
return `${masterQueue}${SCHEDULED_WORKER_QUEUE_SUFFIX}`;
|
||||
}
|
||||
|
||||
return masterQueue;
|
||||
}
|
||||
|
||||
export function parseDisabledWorkerQueues(raw: string | undefined): Set<string> {
|
||||
return new Set(
|
||||
(raw ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean)
|
||||
);
|
||||
}
|
||||
|
||||
export function matchesDisabledWorkerQueue(
|
||||
workerQueue: string,
|
||||
disabledWorkerQueues: ReadonlySet<string>
|
||||
): boolean {
|
||||
if (disabledWorkerQueues.size === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
disabledWorkerQueues.has(workerQueue) || disabledWorkerQueues.has(baseWorkerQueue(workerQueue))
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,759 @@
|
||||
import {
|
||||
type BatchTriggerTaskV2RequestBody,
|
||||
type BatchTriggerTaskV3RequestBody,
|
||||
type BatchTriggerTaskV3Response,
|
||||
type IOPacket,
|
||||
packetRequiresOffloading,
|
||||
parsePacket,
|
||||
TaskRunErrorCodes,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { Evt } from "evt";
|
||||
import { z } from "zod";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
||||
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
||||
import {
|
||||
downloadPacketFromObjectStore,
|
||||
uploadPacketToObjectStore,
|
||||
} from "../../v3/objectStore.server";
|
||||
import type { RunEngine } from "../../v3/runEngine.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { TriggerTaskService } from "../../v3/services/triggerTask.server";
|
||||
import { startActiveSpan } from "../../v3/tracer.server";
|
||||
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
||||
|
||||
const PROCESSING_BATCH_SIZE = 50;
|
||||
const ASYNC_BATCH_PROCESS_SIZE_THRESHOLD = 20;
|
||||
const MAX_ATTEMPTS = 10;
|
||||
|
||||
export const BatchProcessingStrategy = z.enum(["sequential", "parallel"]);
|
||||
export type BatchProcessingStrategy = z.infer<typeof BatchProcessingStrategy>;
|
||||
|
||||
export const BatchProcessingOptions = z.object({
|
||||
batchId: z.string(),
|
||||
processingId: z.string(),
|
||||
range: z.object({ start: z.number().int(), count: z.number().int() }),
|
||||
attemptCount: z.number().int(),
|
||||
strategy: BatchProcessingStrategy,
|
||||
parentRunId: z.string().optional(),
|
||||
resumeParentOnCompletion: z.boolean().optional(),
|
||||
planType: z.string().optional(),
|
||||
});
|
||||
|
||||
export type BatchProcessingOptions = z.infer<typeof BatchProcessingOptions>;
|
||||
|
||||
export type BatchTriggerTaskServiceOptions = {
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
||||
spanParentAsLink?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
realtimeStreamsVersion?: "v1" | "v2";
|
||||
triggerSource?: string;
|
||||
triggerAction?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Larger batches, used in Run Engine v2
|
||||
*/
|
||||
export class RunEngineBatchTriggerService extends WithRunEngine {
|
||||
private _batchProcessingStrategy: BatchProcessingStrategy;
|
||||
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
||||
|
||||
constructor(
|
||||
batchProcessingStrategy?: BatchProcessingStrategy,
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
engine?: RunEngine
|
||||
) {
|
||||
super({ prisma: _prisma, engine });
|
||||
|
||||
// Eric note: We need to force sequential processing because when doing parallel, we end up with high-contention on the parent run lock
|
||||
// becuase we are triggering a lot of runs at once, and each one is trying to lock the parent run.
|
||||
// by forcing sequential, we are only ever locking the parent run for a single run at a time.
|
||||
this._batchProcessingStrategy = "sequential";
|
||||
}
|
||||
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV3RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
): Promise<BatchTriggerTaskV3Response> {
|
||||
try {
|
||||
return await this.traceWithEnv<BatchTriggerTaskV3Response>(
|
||||
"call()",
|
||||
environment,
|
||||
async (span) => {
|
||||
const { friendlyId } = await mintBatchFriendlyId({
|
||||
environment: {
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: body.parentRunId,
|
||||
});
|
||||
|
||||
span.setAttribute("batchId", friendlyId);
|
||||
|
||||
const payloadPacket = await this.#handlePayloadPacket(
|
||||
body.items,
|
||||
`batch/${friendlyId}`,
|
||||
environment
|
||||
);
|
||||
|
||||
const batch = await this.#createAndProcessBatchTaskRun(
|
||||
friendlyId,
|
||||
payloadPacket,
|
||||
environment,
|
||||
body,
|
||||
options
|
||||
);
|
||||
|
||||
if (!batch) {
|
||||
throw new Error("Failed to create batch");
|
||||
}
|
||||
|
||||
return {
|
||||
id: batch.friendlyId,
|
||||
isCached: false,
|
||||
idempotencyKey: batch.idempotencyKey ?? undefined,
|
||||
runCount: body.items.length,
|
||||
};
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
// Detect a prisma transaction Unique constraint violation
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
logger.debug("RunEngineBatchTrigger: Prisma transaction error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
meta: error.meta,
|
||||
});
|
||||
|
||||
if (error.code === "P2002") {
|
||||
const target = error.meta?.target;
|
||||
|
||||
if (
|
||||
Array.isArray(target) &&
|
||||
target.length > 0 &&
|
||||
typeof target[0] === "string" &&
|
||||
target[0].includes("oneTimeUseToken")
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger with a one-time use token as it has already been used."
|
||||
);
|
||||
} else {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger as it has already been triggered with the same idempotency key."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndProcessBatchTaskRun(
|
||||
batchId: string,
|
||||
payloadPacket: IOPacket,
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV2RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
) {
|
||||
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
||||
// validate env existence app-side (covers both create arms below).
|
||||
await controlPlaneResolver.assertEnvExists(environment.id);
|
||||
|
||||
if (body.items.length <= ASYNC_BATCH_PROCESS_SIZE_THRESHOLD) {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: 0,
|
||||
batchSize: PROCESSING_BATCH_SIZE,
|
||||
items: body.items,
|
||||
options,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing complete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: 0,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
});
|
||||
|
||||
// If processing inline does not finish for some reason, enqueue processing the rest of the batch
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][call] Batch inline processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
});
|
||||
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
switch (this._batchProcessingStrategy) {
|
||||
case "sequential": {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: batchId,
|
||||
range: { start: 0, count: PROCESSING_BATCH_SIZE },
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "parallel": {
|
||||
const ranges = Array.from({
|
||||
length: Math.ceil(body.items.length / PROCESSING_BATCH_SIZE),
|
||||
}).map((_, index) => ({
|
||||
start: index * PROCESSING_BATCH_SIZE,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
}));
|
||||
|
||||
await Promise.all(
|
||||
ranges.map((range, index) =>
|
||||
this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: `${index}`,
|
||||
range,
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
|
||||
async #enqueueBatchTaskRun(options: BatchProcessingOptions) {
|
||||
await batchTriggerWorker.enqueue({
|
||||
id: `RunEngineBatchTriggerService.process:${options.batchId}:${options.processingId}`,
|
||||
job: "runengine.processBatchTaskRun",
|
||||
payload: options,
|
||||
});
|
||||
}
|
||||
|
||||
// This is the function that the worker will call
|
||||
async processBatchTaskRun(options: BatchProcessingOptions) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch", {
|
||||
options,
|
||||
});
|
||||
|
||||
const $attemptCount = options.attemptCount + 1;
|
||||
|
||||
if ($attemptCount > MAX_ATTEMPTS) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Max attempts reached", {
|
||||
options,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
// You might want to update the batch status to failed here
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = await this._engine.runStore.findBatchTaskRunById(options.batchId);
|
||||
|
||||
if (!batch) {
|
||||
return;
|
||||
}
|
||||
|
||||
// BatchTaskRun -> RuntimeEnvironment FK is dropped; resolve the env from the scalar id.
|
||||
const environment = await findEnvironmentById(batch.runtimeEnvironmentId);
|
||||
if (!environment) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Environment not found", {
|
||||
batchId: batch.id,
|
||||
runtimeEnvironmentId: batch.runtimeEnvironmentId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.range.start >= batch.runCount) {
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] currentIndex is greater than runCount",
|
||||
{
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
runCount: batch.runCount,
|
||||
attemptCount: $attemptCount,
|
||||
}
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const payloadPacket = await downloadPacketFromObjectStore(
|
||||
{
|
||||
data: batch.payload ?? undefined,
|
||||
dataType: batch.payloadType,
|
||||
},
|
||||
environment
|
||||
);
|
||||
|
||||
const payload = await parsePacket(payloadPacket);
|
||||
|
||||
if (!payload) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Failed to parse payload", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
throw new Error("Failed to parse payload");
|
||||
}
|
||||
|
||||
// Skip zod parsing
|
||||
const $payload = payload as BatchTriggerTaskV2RequestBody["items"];
|
||||
const $options = batch.options as BatchTriggerTaskServiceOptions;
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: options.range.start,
|
||||
batchSize: options.range.count,
|
||||
items: $payload,
|
||||
options: $options,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing complete", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// Only enqueue the next batch task run if the strategy is sequential
|
||||
// if the strategy is parallel, we will already have enqueued the next batch task run
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// if the strategy is sequential, we will requeue processing with a count of the PROCESSING_BATCH_SIZE
|
||||
// if the strategy is parallel, we will requeue processing with a range starting at the workingIndex and a count that is the remainder of this "slice" of the batch
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count, // This will be the same as the original count
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
} else {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
// This will be the remainder of the slice
|
||||
// for example if the original range was 0-50 and the workingIndex is 25, the new range will be 25-25
|
||||
// if the original range was 51-100 and the workingIndex is 75, the new range will be 75-25
|
||||
count: options.range.count - result.workingIndex - options.range.start,
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex,
|
||||
batchSize,
|
||||
items,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
currentIndex: number;
|
||||
batchSize: number;
|
||||
items: BatchTriggerTaskV2RequestBody["items"];
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId?: string | undefined;
|
||||
resumeParentOnCompletion?: boolean | undefined;
|
||||
}): Promise<
|
||||
| { status: "COMPLETE" }
|
||||
| { status: "INCOMPLETE"; workingIndex: number }
|
||||
| { status: "ERROR"; error: string; workingIndex: number }
|
||||
> {
|
||||
// Grab the next PROCESSING_BATCH_SIZE items
|
||||
const itemsToProcess = items.slice(currentIndex, currentIndex + batchSize);
|
||||
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch items", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
runCount: batch.runCount,
|
||||
});
|
||||
|
||||
let workingIndex = currentIndex;
|
||||
|
||||
let runIds: string[] = [];
|
||||
|
||||
const triggerFailedTaskService = new TriggerFailedTaskService({
|
||||
prisma: this._prisma,
|
||||
engine: this._engine,
|
||||
replicaPrisma: this._replica,
|
||||
});
|
||||
|
||||
for (const item of itemsToProcess) {
|
||||
let runFriendlyId: string | null = null;
|
||||
|
||||
try {
|
||||
const run = await this.#processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex: workingIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
if (run) {
|
||||
runFriendlyId = run.friendlyId;
|
||||
}
|
||||
} catch (_error) {
|
||||
// Trigger failed - will try to create pre-failed run below
|
||||
runFriendlyId = null;
|
||||
}
|
||||
|
||||
if (!runFriendlyId) {
|
||||
const errorMessage =
|
||||
"Trigger failed for batch item (queue limit, entitlement, or validation error)";
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Item trigger failed, creating pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
task: item.task,
|
||||
}
|
||||
);
|
||||
|
||||
const failedRunId = await triggerFailedTaskService.call({
|
||||
taskId: item.task,
|
||||
environment,
|
||||
payload: item.payload,
|
||||
payloadType: item.options?.payloadType,
|
||||
errorMessage,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
batch: { id: batch.id, index: workingIndex },
|
||||
// Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
|
||||
// co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
|
||||
runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
|
||||
options: item.options as Record<string, unknown>,
|
||||
traceContext: options?.traceContext as Record<string, unknown> | undefined,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
|
||||
});
|
||||
|
||||
if (failedRunId) {
|
||||
runFriendlyId = failedRunId;
|
||||
} else {
|
||||
logger.error(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Failed to create pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
status: "ERROR",
|
||||
error: "Could not trigger item and could not create pre-failed run",
|
||||
workingIndex,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
runIds.push(runFriendlyId);
|
||||
workingIndex++;
|
||||
}
|
||||
|
||||
const updatedBatch = await this._engine.runStore.updateBatchTaskRun({
|
||||
where: { id: batch.id },
|
||||
data: {
|
||||
runIds: {
|
||||
push: runIds,
|
||||
},
|
||||
processingJobsCount: {
|
||||
increment: runIds.length,
|
||||
},
|
||||
},
|
||||
select: {
|
||||
processingJobsCount: true,
|
||||
runCount: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (updatedBatch.processingJobsCount >= updatedBatch.runCount) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] All runs created", {
|
||||
batchId: batch.friendlyId,
|
||||
processingJobsCount: updatedBatch.processingJobsCount,
|
||||
runCount: updatedBatch.runCount,
|
||||
workingIndex,
|
||||
});
|
||||
|
||||
//if all the runs were idempotent, it's possible the batch is already completed
|
||||
await this._engine.tryCompleteBatch({ batchId: batch.id });
|
||||
}
|
||||
|
||||
// if there are more items to process, requeue the batch
|
||||
if (workingIndex < batch.runCount) {
|
||||
return { status: "INCOMPLETE", workingIndex };
|
||||
}
|
||||
|
||||
return { status: "COMPLETE" };
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
item: BatchTriggerTaskV2RequestBody["items"][number];
|
||||
currentIndex: number;
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId: string | undefined;
|
||||
resumeParentOnCompletion: boolean | undefined;
|
||||
}) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRunItem] Processing item", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
});
|
||||
|
||||
const triggerTaskService = new TriggerTaskService();
|
||||
|
||||
// Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
|
||||
// an org's mint flag flipping between batch creation and this (possibly much later,
|
||||
// worker-processed) item never splits the item from its BatchTaskRun row. See
|
||||
// mintAnchoredRunFriendlyId.server.ts.
|
||||
const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
|
||||
|
||||
const result = await triggerTaskService.call(
|
||||
item.task,
|
||||
environment,
|
||||
{
|
||||
...item,
|
||||
options: {
|
||||
...item.options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
parentBatch: batch.id,
|
||||
},
|
||||
},
|
||||
{
|
||||
triggerVersion: options?.triggerVersion,
|
||||
traceContext: options?.traceContext,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
batchId: batch.id,
|
||||
batchIndex: currentIndex,
|
||||
runFriendlyId,
|
||||
realtimeStreamsVersion: options?.realtimeStreamsVersion,
|
||||
triggerSource: options?.triggerSource ?? "api",
|
||||
triggerAction: options?.triggerAction ?? "trigger",
|
||||
},
|
||||
"V2"
|
||||
);
|
||||
|
||||
return result
|
||||
? {
|
||||
friendlyId: result.run.friendlyId,
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async #handlePayloadPacket(
|
||||
payload: any,
|
||||
pathPrefix: string,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
||||
const packet = { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${pathPrefix}/payload.json`;
|
||||
|
||||
const uploadedFilename = await uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
environment
|
||||
);
|
||||
|
||||
return {
|
||||
data: uploadedFilename,
|
||||
dataType: "application/store",
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import type { InitializeBatchOptions } from "@internal/run-engine";
|
||||
import { type CreateBatchRequestBody, type CreateBatchResponse } from "@trigger.dev/core/v3";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { Evt } from "evt";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { BatchRateLimitExceededError, getBatchLimits } from "../concerns/batchLimits.server";
|
||||
import { DefaultQueueManager } from "../concerns/queues.server";
|
||||
import { DefaultTriggerTaskValidator } from "../validators/triggerTaskValidator";
|
||||
|
||||
export type CreateBatchServiceOptions = {
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
||||
spanParentAsLink?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
realtimeStreamsVersion?: "v1" | "v2";
|
||||
triggerSource?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Create Batch Service (Phase 1 of 2-phase batch API).
|
||||
*
|
||||
* This service handles Phase 1 of the streaming batch API:
|
||||
* 1. Validates entitlement and queue limits
|
||||
* 2. Creates BatchTaskRun in Postgres with status=PENDING, expectedCount set
|
||||
* 3. For batchTriggerAndWait: blocks the parent run immediately
|
||||
* 4. Initializes batch metadata in Redis
|
||||
* 5. Returns batch ID - items are streamed separately via Phase 2
|
||||
*
|
||||
* The batch is NOT sealed until Phase 2 completes.
|
||||
*/
|
||||
export class CreateBatchService extends WithRunEngine {
|
||||
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
||||
private readonly queueConcern: DefaultQueueManager;
|
||||
private readonly validator: DefaultTriggerTaskValidator;
|
||||
|
||||
constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {
|
||||
super({ prisma: _prisma });
|
||||
|
||||
this.queueConcern = new DefaultQueueManager(this._prisma, this._engine, this._replica);
|
||||
this.validator = new DefaultTriggerTaskValidator();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a batch for 2-phase processing.
|
||||
* Items will be streamed separately via the StreamBatchItemsService.
|
||||
*/
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: CreateBatchRequestBody,
|
||||
options: CreateBatchServiceOptions = {}
|
||||
): Promise<CreateBatchResponse> {
|
||||
try {
|
||||
return await this.traceWithEnv<CreateBatchResponse>(
|
||||
"createBatch()",
|
||||
environment,
|
||||
async (span) => {
|
||||
const { id, friendlyId } = await mintBatchFriendlyId({
|
||||
environment: {
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: body.parentRunId,
|
||||
});
|
||||
|
||||
span.setAttribute("batchId", friendlyId);
|
||||
span.setAttribute("runCount", body.runCount);
|
||||
|
||||
const entitlementValidation = await this.validator.validateEntitlement({
|
||||
environment,
|
||||
});
|
||||
|
||||
if (!entitlementValidation.ok) {
|
||||
throw entitlementValidation.error;
|
||||
}
|
||||
|
||||
const planType = entitlementValidation.plan?.type;
|
||||
|
||||
const { config, rateLimiter } = await getBatchLimits(environment.organization);
|
||||
|
||||
// Rate-limit before creating the batch, to stop bursts exceeding the limit.
|
||||
const rateResult = await rateLimiter.limit(environment.id, body.runCount);
|
||||
|
||||
if (!rateResult.success) {
|
||||
throw new BatchRateLimitExceededError(
|
||||
rateResult.limit,
|
||||
rateResult.remaining,
|
||||
new Date(rateResult.reset),
|
||||
body.runCount
|
||||
);
|
||||
}
|
||||
|
||||
// Note: Queue size limits are validated per-queue when batch items are processed,
|
||||
// since we don't know which queues items will go to until they're streamed.
|
||||
|
||||
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
||||
// validate env existence app-side (passthrough when split is off).
|
||||
await controlPlaneResolver.assertEnvExists(environment.id);
|
||||
|
||||
// Created PENDING; sealed (status -> PROCESSING) once items are streamed.
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id,
|
||||
friendlyId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
status: "PENDING",
|
||||
runCount: body.runCount,
|
||||
expectedCount: body.runCount,
|
||||
runIds: [],
|
||||
batchVersion: "runengine:v2", // 2-phase streaming batch API
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
sealed: false,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
// Block parent run if this is a batchTriggerAndWait
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize batch metadata in Redis (without items)
|
||||
const initOptions: InitializeBatchOptions = {
|
||||
batchId: id,
|
||||
friendlyId,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
organizationId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
runCount: body.runCount,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
triggerVersion: options.triggerVersion,
|
||||
traceContext: options.traceContext as Record<string, unknown> | undefined,
|
||||
spanParentAsLink: options.spanParentAsLink,
|
||||
realtimeStreamsVersion: options.realtimeStreamsVersion,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
processingConcurrency: config.processingConcurrency,
|
||||
planType,
|
||||
triggerSource: options.triggerSource,
|
||||
};
|
||||
|
||||
await this._engine.initializeBatch(initOptions);
|
||||
|
||||
logger.info("Batch created", {
|
||||
batchId: friendlyId,
|
||||
runCount: body.runCount,
|
||||
envId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
processingConcurrency: config.processingConcurrency,
|
||||
});
|
||||
|
||||
return {
|
||||
id: friendlyId,
|
||||
runCount: body.runCount,
|
||||
isCached: false,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
};
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
// Handle Prisma unique constraint violations
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
logger.debug("CreateBatchService: Prisma error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
meta: error.meta,
|
||||
});
|
||||
|
||||
if (error.code === "P2002") {
|
||||
const target = error.meta?.target;
|
||||
|
||||
if (
|
||||
Array.isArray(target) &&
|
||||
target.length > 0 &&
|
||||
typeof target[0] === "string" &&
|
||||
target[0].includes("oneTimeUseToken")
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot create batch with a one-time use token as it has already been used."
|
||||
);
|
||||
} else {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot create batch as it has already been created with the same idempotency key."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,837 @@
|
||||
import {
|
||||
type StreamBatchItemsResponse,
|
||||
BatchItemNDJSON as BatchItemNDJSONSchema,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { BatchId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { BatchItem, RunEngine } from "@internal/run-engine";
|
||||
import pMap from "p-map";
|
||||
import type { BatchTaskRunStatus } from "@trigger.dev/database";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { BatchPayloadProcessor } from "../concerns/batchPayloads.server";
|
||||
|
||||
/**
|
||||
* Phase 2 retry idempotency check.
|
||||
*
|
||||
* Returns true when the batch is in a state that means the Phase 2 stream's
|
||||
* job has already been done — every item has a TaskRun record (real or
|
||||
* pre-failed) for the customer to monitor. A retry, or the original call
|
||||
* racing against a fast-completing BatchQueue, should return sealed:true
|
||||
* in these states so the SDK stops retrying.
|
||||
*
|
||||
* Three "work is done" shapes:
|
||||
* - status moved out of PENDING into PROCESSING/COMPLETED/PARTIAL_FAILED
|
||||
* (PROCESSING via our seal, COMPLETED via tryCompleteBatch, PARTIAL_FAILED
|
||||
* via the V2 batchCompletionCallback).
|
||||
* - status stuck at PENDING but `sealed=true`: another concurrent
|
||||
* streamBatchItems call sealed the batch and then the callback's
|
||||
* happy-path branch reset status to PENDING ("all runs created").
|
||||
* - status stuck at PENDING with `sealed=false` but `processingCompletedAt`
|
||||
* set: the cleanup-race. BatchQueue rushed through all items, callback
|
||||
* fired (setting processingCompletedAt), cleanup deleted the Redis
|
||||
* metadata — all before our service got the chance to seal. The work
|
||||
* is done; the discriminator is processingCompletedAt which is set
|
||||
* exclusively by the V2 completion callback.
|
||||
*
|
||||
* ABORTED is excluded — it means ZERO TaskRun records were created (every
|
||||
* per-item attempt failed AND the pre-failed-TaskRun fallback also failed,
|
||||
* or queue-overload on every item). The customer has nothing to monitor
|
||||
* at the run level, so the trigger call must throw to give their retry/
|
||||
* error handling a chance to create a fresh batch.
|
||||
*/
|
||||
export function isIdempotentRetrySuccess(
|
||||
status: BatchTaskRunStatus | null | undefined,
|
||||
sealed: boolean | null | undefined,
|
||||
processingCompletedAt: Date | null | undefined
|
||||
): boolean {
|
||||
return (
|
||||
status === "PROCESSING" ||
|
||||
status === "COMPLETED" ||
|
||||
status === "PARTIAL_FAILED" ||
|
||||
(status === "PENDING" && (sealed === true || processingCompletedAt != null))
|
||||
);
|
||||
}
|
||||
|
||||
export type StreamBatchItemsServiceOptions = {
|
||||
maxItemBytes: number;
|
||||
/** Max items processed concurrently. The route wires this to STREAMING_BATCH_INGEST_CONCURRENCY. */
|
||||
concurrency: number;
|
||||
};
|
||||
|
||||
export type OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED";
|
||||
index: number;
|
||||
task: string;
|
||||
actualSize: number;
|
||||
maxSize: number;
|
||||
};
|
||||
|
||||
export type StreamBatchItemsServiceConstructorOptions = {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
engine?: RunEngine;
|
||||
/** Override the payload processor (used in tests to observe ingest concurrency). */
|
||||
payloadProcessor?: BatchPayloadProcessor;
|
||||
};
|
||||
|
||||
/**
|
||||
* Stream Batch Items Service (Phase 2 of 2-phase batch API).
|
||||
*
|
||||
* This service handles Phase 2 of the streaming batch API:
|
||||
* 1. Validates batch exists and is in PENDING status
|
||||
* 2. Processes NDJSON stream item by item
|
||||
* 3. Calls engine.enqueueBatchItem() for each item
|
||||
* 4. Tracks accepted/deduplicated counts
|
||||
* 5. On completion: validates count, seals the batch
|
||||
*
|
||||
* The service is designed for streaming and processes items as they arrive,
|
||||
* providing backpressure through the async iterator pattern.
|
||||
*/
|
||||
export class StreamBatchItemsService extends WithRunEngine {
|
||||
private readonly payloadProcessor: BatchPayloadProcessor;
|
||||
|
||||
constructor(opts: StreamBatchItemsServiceConstructorOptions = {}) {
|
||||
super({ prisma: opts.prisma ?? prisma, engine: opts.engine });
|
||||
this.payloadProcessor = opts.payloadProcessor ?? new BatchPayloadProcessor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a batch friendly ID to its internal ID format.
|
||||
* Throws a ServiceValidationError with 400 status if the ID is malformed.
|
||||
*/
|
||||
private parseBatchFriendlyId(friendlyId: string): string {
|
||||
try {
|
||||
return BatchId.fromFriendlyId(friendlyId);
|
||||
} catch {
|
||||
throw new ServiceValidationError(`Invalid batchFriendlyId: ${friendlyId}`, 400);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a stream of batch items from an async iterator.
|
||||
* Each item is validated and enqueued to the BatchQueue.
|
||||
* The batch is sealed when the stream completes.
|
||||
*/
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
batchFriendlyId: string,
|
||||
itemsIterator: AsyncIterable<unknown>,
|
||||
options: StreamBatchItemsServiceOptions
|
||||
): Promise<StreamBatchItemsResponse> {
|
||||
return this.traceWithEnv<StreamBatchItemsResponse>(
|
||||
"streamBatchItems()",
|
||||
environment,
|
||||
async (span) => {
|
||||
span.setAttribute("batchId", batchFriendlyId);
|
||||
|
||||
// Convert friendly ID to internal ID
|
||||
const batchId = this.parseBatchFriendlyId(batchFriendlyId);
|
||||
|
||||
// Validate batch exists and belongs to this environment. Routed by batch id so a
|
||||
// run-ops id (NEW-resident) batch is found on the owning DB; the env-ownership check that
|
||||
// was in the where clause is enforced app-side below.
|
||||
const batch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (!batch || batch.runtimeEnvironmentId !== environment.id) {
|
||||
throw new ServiceValidationError(`Batch ${batchFriendlyId} not found`);
|
||||
}
|
||||
|
||||
if (isIdempotentRetrySuccess(batch.status, batch.sealed, batch.processingCompletedAt)) {
|
||||
logger.info("Batch already sealed/completed - treating Phase 2 retry as success", {
|
||||
batchId: batchFriendlyId,
|
||||
batchSealed: batch.sealed,
|
||||
batchStatus: batch.status,
|
||||
processingCompletedAt: batch.processingCompletedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted: 0,
|
||||
itemsDeduplicated: 0,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
if (batch.status !== "PENDING") {
|
||||
// ABORTED or any other unexpected non-PENDING state — surface as an error.
|
||||
// For ABORTED specifically, throwing is required so the customer's
|
||||
// batchTrigger() retries (a new batch) can recreate the runs.
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is not in PENDING status (current: ${batch.status})`
|
||||
);
|
||||
}
|
||||
|
||||
// Process items from the stream with bounded concurrency.
|
||||
//
|
||||
// Ordering and idempotency do NOT depend on processing order:
|
||||
// - The BatchQueue derives run order from each item's index
|
||||
// (enqueue timestamp = batch.createdAt + itemIndex), not enqueue order.
|
||||
// - enqueueBatchItem() dedups atomically per index.
|
||||
// We cap concurrency to bound peak in-flight memory (≈ concurrency ×
|
||||
// maxItemBytes) and to keep backpressure on the request body stream.
|
||||
// p-map pulls lazily from the async iterator — at most `concurrency`
|
||||
// items are read and in flight at once. stopOnError aborts ingestion on
|
||||
// the first failure (the batch is left unsealed; the SDK's retry
|
||||
// re-streams and dedups already-enqueued items).
|
||||
const outcomes = await pMap(
|
||||
itemsIterator,
|
||||
(rawItem) => this.#processItem(rawItem, batchId, environment, batch.runCount),
|
||||
{ concurrency: options.concurrency, stopOnError: true }
|
||||
);
|
||||
|
||||
let itemsAccepted = 0;
|
||||
let itemsDeduplicated = 0;
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome === "accepted") {
|
||||
itemsAccepted++;
|
||||
} else {
|
||||
itemsDeduplicated++;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the actual enqueued count from Redis
|
||||
const enqueuedCount = await this._engine.getBatchEnqueuedCount(batchId);
|
||||
|
||||
// Validate we received the expected number of items
|
||||
if (enqueuedCount !== batch.runCount) {
|
||||
// The batch queue consumers may have already processed all items and
|
||||
// cleaned up the Redis keys before we got here. This happens when all
|
||||
// runs complete fast enough that cleanup() deletes the enqueuedItemsKey
|
||||
// before we read it — typically when the last item executes in the
|
||||
// milliseconds between the loop ending and getBatchEnqueuedCount() being called.
|
||||
// Check both sealed (sealed by this endpoint on a concurrent request) and
|
||||
// COMPLETED (sealed by the BatchQueue completion path before we got here).
|
||||
const currentBatch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (
|
||||
isIdempotentRetrySuccess(
|
||||
currentBatch?.status,
|
||||
currentBatch?.sealed,
|
||||
currentBatch?.processingCompletedAt
|
||||
)
|
||||
) {
|
||||
logger.info("Batch already sealed before count check (fast completion)", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
enqueuedCount,
|
||||
expectedCount: batch.runCount,
|
||||
batchStatus: currentBatch?.status,
|
||||
processingCompletedAt: currentBatch?.processingCompletedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
if (currentBatch?.status === "ABORTED") {
|
||||
// Zero TaskRuns exist — the count-mismatch sealed:false semantics
|
||||
// ("retry with missing items") would mislead the SDK. Throw so the
|
||||
// customer's batchTrigger() retry creates a fresh batch.
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is not in PENDING status (current: ABORTED)`
|
||||
);
|
||||
}
|
||||
|
||||
logger.warn("Batch item count mismatch", {
|
||||
batchId: batchFriendlyId,
|
||||
expected: batch.runCount,
|
||||
received: enqueuedCount,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
});
|
||||
|
||||
// Don't seal the batch if count doesn't match
|
||||
// Return sealed: false so client knows to retry with missing items
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: false,
|
||||
enqueuedCount,
|
||||
expectedCount: batch.runCount,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
// Seal the batch - use conditional update to prevent TOCTOU race
|
||||
// Another concurrent request may have already sealed this batch
|
||||
const now = new Date();
|
||||
const sealResult = await this._engine.runStore.updateManyBatchTaskRun({
|
||||
where: {
|
||||
id: batchId,
|
||||
sealed: false,
|
||||
status: "PENDING",
|
||||
},
|
||||
data: {
|
||||
sealed: true,
|
||||
sealedAt: now,
|
||||
status: "PROCESSING",
|
||||
processingStartedAt: now,
|
||||
},
|
||||
});
|
||||
|
||||
// Check if we won the race to seal the batch
|
||||
if (sealResult.count === 0) {
|
||||
// The conditional update failed because the batch was no longer in
|
||||
// PENDING status. Re-query to determine which path got there first:
|
||||
// - A concurrent streaming request already sealed and moved it to
|
||||
// PROCESSING.
|
||||
// - The BatchQueue completion path finished all runs and set it to
|
||||
// COMPLETED (without setting sealed=true — that's this endpoint's
|
||||
// job). This window exists between completionCallback (which calls
|
||||
// tryCompleteBatch) and cleanup() in BatchQueue — see
|
||||
// batch-queue/index.ts.
|
||||
// Either way the goal — a durable batch that the SDK stops retrying —
|
||||
// has been achieved, so we return sealed: true.
|
||||
const currentBatch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (
|
||||
isIdempotentRetrySuccess(
|
||||
currentBatch?.status,
|
||||
currentBatch?.sealed,
|
||||
currentBatch?.processingCompletedAt
|
||||
)
|
||||
) {
|
||||
logger.info("Batch already sealed/completed by concurrent path", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
envId: environment.id,
|
||||
batchStatus: currentBatch?.status,
|
||||
batchSealed: currentBatch?.sealed,
|
||||
processingCompletedAt: currentBatch?.processingCompletedAt,
|
||||
});
|
||||
|
||||
span.setAttribute("itemsAccepted", itemsAccepted);
|
||||
span.setAttribute("itemsDeduplicated", itemsDeduplicated);
|
||||
span.setAttribute("sealedByConcurrentRequest", true);
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
// Batch is in an unexpected state - fail with error
|
||||
const actualStatus = currentBatch?.status ?? "unknown";
|
||||
const actualSealed = currentBatch?.sealed ?? "unknown";
|
||||
logger.error("Batch seal race condition: unexpected state", {
|
||||
batchId: batchFriendlyId,
|
||||
expectedStatus: "PENDING",
|
||||
actualStatus,
|
||||
expectedSealed: false,
|
||||
actualSealed,
|
||||
envId: environment.id,
|
||||
});
|
||||
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is in unexpected state (status: ${actualStatus}, sealed: ${actualSealed}). Cannot seal batch.`
|
||||
);
|
||||
}
|
||||
|
||||
logger.info("Batch sealed and ready for processing", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
totalEnqueued: enqueuedCount,
|
||||
envId: environment.id,
|
||||
});
|
||||
|
||||
span.setAttribute("itemsAccepted", itemsAccepted);
|
||||
span.setAttribute("itemsDeduplicated", itemsDeduplicated);
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single streamed batch item: validate it, offload its payload to
|
||||
* object storage if oversized, and enqueue it. Returns whether the item was
|
||||
* newly enqueued ("accepted") or was a duplicate ("deduplicated"). Throws
|
||||
* ServiceValidationError for invalid items, which aborts the stream.
|
||||
*
|
||||
* Safe to run concurrently: enqueueBatchItem() is atomic and order-independent
|
||||
* per item index, and each item carries its own index (real items from the
|
||||
* SDK; oversized markers are stamped by the NDJSON parser).
|
||||
*/
|
||||
async #processItem(
|
||||
rawItem: unknown,
|
||||
batchId: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
runCount: number
|
||||
): Promise<"accepted" | "deduplicated"> {
|
||||
// Oversized item marker emitted by the NDJSON parser
|
||||
if (rawItem && typeof rawItem === "object" && "__batchItemError" in rawItem) {
|
||||
const marker = rawItem as OversizedItemMarker;
|
||||
|
||||
// Same out-of-range guard as normal items: an oversized item with an
|
||||
// out-of-range index must 4xx rather than create a stray pre-failed run.
|
||||
if (marker.index >= runCount) {
|
||||
throw new ServiceValidationError(
|
||||
`Item index ${marker.index} exceeds batch runCount ${runCount}`
|
||||
);
|
||||
}
|
||||
|
||||
const errorMessage = `Batch item payload is too large (${(marker.actualSize / 1024).toFixed(
|
||||
1
|
||||
)} KB). Maximum allowed size is ${(marker.maxSize / 1024).toFixed(
|
||||
1
|
||||
)} KB. Reduce the payload size or offload large data to external storage.`;
|
||||
|
||||
// Enqueue with __error metadata - processItemCallback will detect this
|
||||
// and use TriggerFailedTaskService to create a pre-failed run
|
||||
const batchItem: BatchItem = {
|
||||
task: marker.task,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
options: {
|
||||
__error: errorMessage,
|
||||
__errorCode: "PAYLOAD_TOO_LARGE",
|
||||
},
|
||||
};
|
||||
|
||||
const result = await this._engine.enqueueBatchItem(
|
||||
batchId,
|
||||
environment.id,
|
||||
marker.index,
|
||||
batchItem
|
||||
);
|
||||
|
||||
return result.enqueued ? "accepted" : "deduplicated";
|
||||
}
|
||||
|
||||
// Parse and validate the item
|
||||
const parseResult = BatchItemNDJSONSchema.safeParse(rawItem);
|
||||
if (!parseResult.success) {
|
||||
const rawIndex = (rawItem as { index?: unknown } | null)?.index;
|
||||
const where = typeof rawIndex === "number" ? `index ${rawIndex}` : "unknown index";
|
||||
throw new ServiceValidationError(`Invalid item at ${where}: ${parseResult.error.message}`);
|
||||
}
|
||||
|
||||
const item = parseResult.data;
|
||||
|
||||
// Validate index is within expected range
|
||||
if (item.index >= runCount) {
|
||||
throw new ServiceValidationError(
|
||||
`Item index ${item.index} exceeds batch runCount ${runCount}`
|
||||
);
|
||||
}
|
||||
|
||||
// Get the original payload type
|
||||
const originalPayloadType = (item.options?.payloadType as string) ?? "application/json";
|
||||
|
||||
// Process payload - offload to object storage if it exceeds threshold
|
||||
const processedPayload = await this.payloadProcessor.process(
|
||||
item.payload,
|
||||
originalPayloadType,
|
||||
batchId,
|
||||
item.index,
|
||||
environment
|
||||
);
|
||||
|
||||
// Convert to BatchItem format with potentially offloaded payload
|
||||
const batchItem: BatchItem = {
|
||||
task: item.task,
|
||||
payload: processedPayload.payload,
|
||||
payloadType: processedPayload.payloadType,
|
||||
options: item.options,
|
||||
};
|
||||
|
||||
// Enqueue the item
|
||||
const result = await this._engine.enqueueBatchItem(
|
||||
batchId,
|
||||
environment.id,
|
||||
item.index,
|
||||
batchItem
|
||||
);
|
||||
|
||||
return result.enqueued ? "accepted" : "deduplicated";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `index` and `task` from raw JSON bytes without decoding the full line.
|
||||
* Scans at most 512 bytes, tracking JSON nesting depth to only match top-level keys.
|
||||
*/
|
||||
export function extractIndexAndTask(bytes: Uint8Array): { index: number; task: string } {
|
||||
let index = -1;
|
||||
let task = "unknown";
|
||||
let depth = 0;
|
||||
let foundIndex = false;
|
||||
let foundTask = false;
|
||||
const limit = Math.min(bytes.byteLength, 512);
|
||||
|
||||
const QUOTE = 0x22; // "
|
||||
const COLON = 0x3a; // :
|
||||
const LBRACE = 0x7b; // {
|
||||
const RBRACE = 0x7d; // }
|
||||
const LBRACKET = 0x5b; // [
|
||||
const RBRACKET = 0x5d; // ]
|
||||
const BACKSLASH = 0x5c; // \
|
||||
|
||||
// Byte patterns for "index" and "task" (without quotes)
|
||||
const INDEX_BYTES = [0x69, 0x6e, 0x64, 0x65, 0x78]; // index
|
||||
const TASK_BYTES = [0x74, 0x61, 0x73, 0x6b]; // task
|
||||
|
||||
let i = 0;
|
||||
while (i < limit && !(foundIndex && foundTask)) {
|
||||
const b = bytes[i];
|
||||
|
||||
if (b === LBRACE || b === LBRACKET) {
|
||||
depth++;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (b === RBRACE || b === RBRACKET) {
|
||||
depth--;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only match keys at depth 1 (top-level object)
|
||||
if (b === QUOTE && depth === 1) {
|
||||
// Read the key inside quotes
|
||||
const keyStart = i + 1;
|
||||
let keyEnd = keyStart;
|
||||
while (keyEnd < limit && bytes[keyEnd] !== QUOTE) {
|
||||
if (bytes[keyEnd] === BACKSLASH) keyEnd++; // skip escaped char
|
||||
keyEnd++;
|
||||
}
|
||||
|
||||
const keyLen = keyEnd - keyStart;
|
||||
|
||||
// Check if this key matches "index" or "task"
|
||||
const isIndex =
|
||||
!foundIndex &&
|
||||
keyLen === INDEX_BYTES.length &&
|
||||
INDEX_BYTES.every((b, j) => bytes[keyStart + j] === b);
|
||||
const isTask =
|
||||
!foundTask &&
|
||||
keyLen === TASK_BYTES.length &&
|
||||
TASK_BYTES.every((b, j) => bytes[keyStart + j] === b);
|
||||
|
||||
if (isIndex || isTask) {
|
||||
// Skip past closing quote and find colon
|
||||
let pos = keyEnd + 1;
|
||||
while (pos < limit && bytes[pos] !== COLON) pos++;
|
||||
pos++; // skip colon
|
||||
// Skip whitespace
|
||||
while (pos < limit && (bytes[pos] === 0x20 || bytes[pos] === 0x09)) pos++;
|
||||
|
||||
if (isIndex) {
|
||||
// Parse digits
|
||||
let num = 0;
|
||||
let hasDigit = false;
|
||||
while (pos < limit && bytes[pos] >= 0x30 && bytes[pos] <= 0x39) {
|
||||
num = num * 10 + (bytes[pos] - 0x30);
|
||||
hasDigit = true;
|
||||
pos++;
|
||||
}
|
||||
if (hasDigit) {
|
||||
index = num;
|
||||
foundIndex = true;
|
||||
}
|
||||
} else {
|
||||
// Parse quoted string value
|
||||
if (pos < limit && bytes[pos] === QUOTE) {
|
||||
const valStart = pos + 1;
|
||||
let valEnd = valStart;
|
||||
while (valEnd < limit && bytes[valEnd] !== QUOTE) {
|
||||
if (bytes[valEnd] === BACKSLASH) valEnd++;
|
||||
valEnd++;
|
||||
}
|
||||
// Decode just this slice
|
||||
try {
|
||||
task = new TextDecoder("utf-8", { fatal: true }).decode(
|
||||
bytes.slice(valStart, valEnd)
|
||||
);
|
||||
foundTask = true;
|
||||
} catch {
|
||||
// Leave as "unknown"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip past the key's closing quote
|
||||
i = keyEnd + 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
return { index, task };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an NDJSON parser transform stream.
|
||||
*
|
||||
* Converts a stream of Uint8Array chunks into parsed JSON objects.
|
||||
* Each line in the NDJSON is parsed independently.
|
||||
*
|
||||
* Uses byte-buffer accumulation to:
|
||||
* - Prevent OOM from unbounded string buffers
|
||||
* - Properly handle multibyte UTF-8 characters across chunk boundaries
|
||||
* - Check size limits on raw bytes before decoding
|
||||
*
|
||||
* @param maxItemBytes - Maximum allowed bytes per line (item)
|
||||
* @returns TransformStream that outputs parsed JSON objects
|
||||
*/
|
||||
export function createNdjsonParserStream(
|
||||
maxItemBytes: number
|
||||
): TransformStream<Uint8Array, unknown> {
|
||||
// Single decoder instance, reused for all lines
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
|
||||
// Byte buffer: array of chunks with tracked total length
|
||||
let chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
let lineNumber = 0;
|
||||
// 0-based position of the next object we emit (parsed item or oversized
|
||||
// marker). The parser is the single sequential point in the pipeline, so this
|
||||
// is the authoritative source of item ordering — downstream consumers can
|
||||
// process items concurrently and must not rely on processing order to derive
|
||||
// an item's index. Used to back-fill an oversized marker's index when it
|
||||
// couldn't be extracted from the (truncated) raw bytes.
|
||||
let emittedCount = 0;
|
||||
// When an oversized incomplete line is detected (Case 2), we must discard
|
||||
// all remaining bytes of that line until the next newline delimiter.
|
||||
let skipUntilNewline = false;
|
||||
|
||||
const NEWLINE_BYTE = 0x0a; // '\n'
|
||||
|
||||
/**
|
||||
* Emit a parsed object or marker downstream and advance the emit position.
|
||||
* Every emitted object MUST go through here so `emittedCount` stays aligned
|
||||
* with item position (empty/skipped lines never emit, so they don't count).
|
||||
*/
|
||||
function emit(controller: TransformStreamDefaultController<unknown>, obj: unknown): void {
|
||||
controller.enqueue(obj);
|
||||
emittedCount++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate all chunks into a single Uint8Array
|
||||
*/
|
||||
function concatenateChunks(): Uint8Array {
|
||||
if (chunks.length === 0) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
if (chunks.length === 1) {
|
||||
return chunks[0];
|
||||
}
|
||||
const result = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the index of the first newline byte in the buffer.
|
||||
* Returns -1 if not found.
|
||||
*/
|
||||
function findNewlineIndex(): number {
|
||||
let globalIndex = 0;
|
||||
for (const chunk of chunks) {
|
||||
for (let i = 0; i < chunk.byteLength; i++) {
|
||||
if (chunk[i] === NEWLINE_BYTE) {
|
||||
return globalIndex + i;
|
||||
}
|
||||
}
|
||||
globalIndex += chunk.byteLength;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract bytes from the buffer up to (but not including) the given index,
|
||||
* and remove those bytes plus the delimiter from the buffer.
|
||||
*/
|
||||
function extractLine(newlineIndex: number): Uint8Array {
|
||||
const fullBuffer = concatenateChunks();
|
||||
const lineBytes = fullBuffer.slice(0, newlineIndex);
|
||||
const remaining = fullBuffer.slice(newlineIndex + 1); // Skip the newline
|
||||
|
||||
// Reset buffer with remaining bytes
|
||||
if (remaining.byteLength > 0) {
|
||||
chunks = [remaining];
|
||||
totalBytes = remaining.byteLength;
|
||||
} else {
|
||||
chunks = [];
|
||||
totalBytes = 0;
|
||||
}
|
||||
|
||||
return lineBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a line from bytes, handling whitespace trimming.
|
||||
* Returns the parsed object or null for empty lines.
|
||||
*/
|
||||
function parseLine(
|
||||
lineBytes: Uint8Array,
|
||||
controller: TransformStreamDefaultController<unknown>
|
||||
): void {
|
||||
lineNumber++;
|
||||
|
||||
// Decode the line bytes (stream: false since this is a complete line)
|
||||
let lineText: string;
|
||||
try {
|
||||
lineText = decoder.decode(lineBytes, { stream: false });
|
||||
} catch (err) {
|
||||
throw new Error(`Invalid UTF-8 at line ${lineNumber}: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const trimmed = lineText.trim();
|
||||
if (!trimmed) {
|
||||
return; // Skip empty lines
|
||||
}
|
||||
|
||||
try {
|
||||
const obj = JSON.parse(trimmed);
|
||||
emit(controller, obj);
|
||||
} catch (err) {
|
||||
throw new Error(`Invalid JSON at line ${lineNumber}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return new TransformStream<Uint8Array, unknown>({
|
||||
transform(chunk, controller) {
|
||||
// If we're skipping the remainder of an oversized line, scan for the
|
||||
// next newline in this chunk and discard everything before it.
|
||||
if (skipUntilNewline) {
|
||||
const nlPos = chunk.indexOf(NEWLINE_BYTE);
|
||||
if (nlPos === -1) {
|
||||
// Entire chunk is still part of the oversized line — discard it
|
||||
return;
|
||||
}
|
||||
// Found the newline — keep everything after it
|
||||
skipUntilNewline = false;
|
||||
const remaining = chunk.slice(nlPos + 1);
|
||||
if (remaining.byteLength === 0) {
|
||||
return;
|
||||
}
|
||||
// Replace chunk with the remainder and fall through to normal processing
|
||||
chunk = remaining;
|
||||
}
|
||||
|
||||
// Append chunk to buffer
|
||||
chunks.push(chunk);
|
||||
totalBytes += chunk.byteLength;
|
||||
|
||||
// Process all complete lines in the buffer
|
||||
let newlineIndex: number;
|
||||
while ((newlineIndex = findNewlineIndex()) !== -1) {
|
||||
// Check size limit BEFORE extracting/decoding (bytes up to newline)
|
||||
if (newlineIndex > maxItemBytes) {
|
||||
// Case 1: Complete line exceeds limit - emit marker instead of throwing
|
||||
const lineBytes = extractLine(newlineIndex);
|
||||
const extracted = extractIndexAndTask(lineBytes);
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: newlineIndex,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
lineNumber++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const lineBytes = extractLine(newlineIndex);
|
||||
parseLine(lineBytes, controller);
|
||||
}
|
||||
|
||||
// Check if the remaining buffer (incomplete line) exceeds the limit
|
||||
// This prevents OOM from a single huge line without newlines
|
||||
if (totalBytes > maxItemBytes) {
|
||||
// Case 2: Incomplete line exceeds limit - emit marker instead of throwing
|
||||
const extracted = extractIndexAndTask(concatenateChunks());
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: totalBytes,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
lineNumber++;
|
||||
// Clear buffer and skip remaining bytes of this oversized line
|
||||
// until the next newline delimiter is found in a subsequent chunk
|
||||
chunks = [];
|
||||
totalBytes = 0;
|
||||
skipUntilNewline = true;
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
// Flush any remaining bytes from the decoder's internal state
|
||||
// This handles multibyte characters that may have been split across chunks
|
||||
decoder.decode(new Uint8Array(0), { stream: false });
|
||||
|
||||
// Process any remaining buffered data (no trailing newline case)
|
||||
if (totalBytes === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check size limit before processing final line
|
||||
if (totalBytes > maxItemBytes) {
|
||||
// Case 3: Flush with oversized remaining - emit marker instead of throwing
|
||||
const extracted = extractIndexAndTask(concatenateChunks());
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: totalBytes,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
return;
|
||||
}
|
||||
|
||||
const finalBytes = concatenateChunks();
|
||||
parseLine(finalBytes, controller);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a ReadableStream into an AsyncIterable.
|
||||
* Useful for processing streams with for-await-of loops.
|
||||
*/
|
||||
export async function* streamToAsyncIterable<T>(stream: ReadableStream<T>): AsyncIterable<T> {
|
||||
const reader = stream.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
yield value;
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
// Empty singletons satisfy the module-level wiring imports; the mint method under test is driven
|
||||
// directly via (service as any) and never touches the DB (same boundary as triggerTask.server.test.ts).
|
||||
vi.mock("~/db.server", () => ({
|
||||
prisma: {},
|
||||
$replica: {},
|
||||
runOpsNewPrisma: {},
|
||||
runOpsLegacyPrisma: {},
|
||||
runOpsNewReplica: {},
|
||||
runOpsLegacyReplica: {},
|
||||
}));
|
||||
vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
|
||||
|
||||
import { classifyKind, generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
||||
|
||||
function buildService() {
|
||||
return new TriggerFailedTaskService({ prisma: {} as any, engine: {} as any });
|
||||
}
|
||||
|
||||
describe("TriggerFailedTaskService.mintFailedRunFriendlyId", () => {
|
||||
it("returns the caller-supplied runFriendlyId verbatim (override wins over any mint)", async () => {
|
||||
const override = RunId.toFriendlyId(generateRunOpsId());
|
||||
const minted = await (buildService() as any).mintFailedRunFriendlyId({
|
||||
organizationId: "org_1",
|
||||
environmentId: "env_1",
|
||||
runFriendlyId: override,
|
||||
});
|
||||
expect(minted).toBe(override);
|
||||
});
|
||||
|
||||
it("without an override, still inherits a run-ops (NEW) parent by id-shape", async () => {
|
||||
const parentRunFriendlyId = RunId.toFriendlyId(generateRunOpsId());
|
||||
const minted = await (buildService() as any).mintFailedRunFriendlyId({
|
||||
organizationId: "org_1",
|
||||
environmentId: "env_1",
|
||||
parentRunFriendlyId,
|
||||
});
|
||||
expect(classifyKind(minted)).toBe("runOpsId");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,435 @@
|
||||
import type { RunEngine } from "@internal/run-engine";
|
||||
import { TaskRunErrorCodes, type TaskRunError } from "@trigger.dev/core/v3";
|
||||
import { RunId, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type {
|
||||
PrismaClientOrTransaction,
|
||||
RuntimeEnvironmentType,
|
||||
TaskRun,
|
||||
} from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
|
||||
import { getEventRepository } from "~/v3/eventRepository/index.server";
|
||||
import { runStore as defaultRunStore } from "~/v3/runStore.server";
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import type { IEventRepository } from "~/v3/eventRepository/eventRepository.types";
|
||||
import { PerformTaskRunAlertsService } from "~/v3/services/alerts/performTaskRunAlerts.server";
|
||||
import { DefaultQueueManager } from "../concerns/queues.server";
|
||||
import type { TriggerTaskRequest } from "../types";
|
||||
|
||||
export type TriggerFailedTaskRequest = {
|
||||
/** The task identifier (e.g. "my-task") */
|
||||
taskId: string;
|
||||
/** The fully-resolved authenticated environment */
|
||||
environment: AuthenticatedEnvironment;
|
||||
/** Raw payload — string or object */
|
||||
payload: unknown;
|
||||
/** MIME type of the payload (defaults to "application/json") */
|
||||
payloadType?: string;
|
||||
/** Error message describing why the run failed */
|
||||
errorMessage: string;
|
||||
/** Parent run friendly ID (e.g. "run_xxxx") */
|
||||
parentRunId?: string;
|
||||
/** Whether completing this run should resume the parent */
|
||||
resumeParentOnCompletion?: boolean;
|
||||
/** Batch association */
|
||||
batch?: { id: string; index: number };
|
||||
/** Trigger options from the original request (queue config, etc.) */
|
||||
options?: Record<string, unknown>;
|
||||
/** Trace context for span correlation */
|
||||
traceContext?: Record<string, unknown>;
|
||||
/** Whether the span parent should be treated as a link rather than a parent */
|
||||
spanParentAsLink?: boolean;
|
||||
|
||||
errorCode?: TaskRunErrorCodes;
|
||||
|
||||
/** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
|
||||
runFriendlyId?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a pre-failed TaskRun with a trace event span.
|
||||
*
|
||||
* This is used when a task cannot be triggered (e.g. queue limit reached, validation
|
||||
* error, etc.) but we still need to record the failure so that:
|
||||
* - Batch completion can track the item
|
||||
* - Parent runs get unblocked
|
||||
* - The failed run shows up in the run logs view
|
||||
*
|
||||
* This service resolves the parent run (for rootTaskRunId/depth) and queue properties
|
||||
* the same way triggerTask does, so the run is correctly associated in the task tree
|
||||
* and the SpanPresenter can find the TaskQueue.
|
||||
*/
|
||||
export class TriggerFailedTaskService {
|
||||
private readonly prisma: PrismaClientOrTransaction;
|
||||
private readonly replicaPrisma: PrismaClientOrTransaction;
|
||||
private readonly engine: RunEngine;
|
||||
// Resolves the parent run for depth/root/parent linkage. Defaults to the shared
|
||||
// singleton (in production the same store the engine writes through). Injected in
|
||||
// tests so the read resolves on the same store the engine wrote to.
|
||||
private readonly runStore: RunStore;
|
||||
// Defaults to getEventRepository's org-flag resolution, which reads through the
|
||||
// global prisma client; tests inject a repository bound to their testcontainer DB.
|
||||
private readonly eventRepository?: { repository: IEventRepository; store: string };
|
||||
|
||||
constructor(opts: {
|
||||
prisma: PrismaClientOrTransaction;
|
||||
engine: RunEngine;
|
||||
replicaPrisma?: PrismaClientOrTransaction;
|
||||
runStore?: RunStore;
|
||||
eventRepository?: { repository: IEventRepository; store: string };
|
||||
}) {
|
||||
this.prisma = opts.prisma;
|
||||
this.replicaPrisma = opts.replicaPrisma ?? opts.prisma;
|
||||
this.engine = opts.engine;
|
||||
this.runStore = opts.runStore ?? defaultRunStore;
|
||||
this.eventRepository = opts.eventRepository;
|
||||
}
|
||||
|
||||
// Mint a failed run's friendlyId. The id-kind decides which store the run is
|
||||
// born in (cuid → legacy store, run-ops id → new store); the whole subgraph of a
|
||||
// run must agree. A caller-supplied runFriendlyId (batch-anchored id) wins verbatim;
|
||||
// otherwise root failed runs mint by the environment's setting and child failed runs
|
||||
// inherit the parent's current store so they never split.
|
||||
private async mintFailedRunFriendlyId(args: {
|
||||
organizationId: string;
|
||||
environmentId: string;
|
||||
orgFeatureFlags?: unknown;
|
||||
parentRunFriendlyId?: string;
|
||||
runFriendlyId?: string;
|
||||
}): Promise<string> {
|
||||
if (args.runFriendlyId) {
|
||||
return args.runFriendlyId;
|
||||
}
|
||||
|
||||
const mintKind = args.parentRunFriendlyId
|
||||
? resolveInheritedMintKind(args.parentRunFriendlyId)
|
||||
: await resolveRunIdMintKind({
|
||||
organizationId: args.organizationId,
|
||||
id: args.environmentId,
|
||||
orgFeatureFlags: args.orgFeatureFlags,
|
||||
});
|
||||
|
||||
return mintKind === "runOpsId"
|
||||
? RunId.toFriendlyId(generateRunOpsId())
|
||||
: RunId.generate().friendlyId;
|
||||
}
|
||||
|
||||
async call(request: TriggerFailedTaskRequest): Promise<string | null> {
|
||||
const taskRunError: TaskRunError = {
|
||||
type: "INTERNAL_ERROR" as const,
|
||||
code: request.errorCode ?? TaskRunErrorCodes.UNSPECIFIED_ERROR,
|
||||
message: request.errorMessage,
|
||||
};
|
||||
|
||||
// Held for the catch's log line; the in-try `const` is what consumers use.
|
||||
let mintedFriendlyId: string | undefined;
|
||||
|
||||
try {
|
||||
// Mint inside the try: classifying a user-supplied parentRunId throws on
|
||||
// an unclassifiable id, so keep it within the catch's null-return contract.
|
||||
const failedRunFriendlyId = await this.mintFailedRunFriendlyId({
|
||||
organizationId: request.environment.organizationId,
|
||||
environmentId: request.environment.id,
|
||||
orgFeatureFlags: request.environment.organization.featureFlags,
|
||||
parentRunFriendlyId: request.parentRunId,
|
||||
runFriendlyId: request.runFriendlyId,
|
||||
});
|
||||
mintedFriendlyId = failedRunFriendlyId;
|
||||
|
||||
const { repository, store } =
|
||||
this.eventRepository ??
|
||||
(await getEventRepository(
|
||||
request.environment.organization.id,
|
||||
request.environment.organization.featureFlags as Record<string, unknown>,
|
||||
undefined
|
||||
));
|
||||
|
||||
// Resolve parent run for rootTaskRunId and depth (same as triggerTask.server.ts)
|
||||
const parentRun = request.parentRunId
|
||||
? await this.runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(request.parentRunId),
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
},
|
||||
this.prisma
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const depth = parentRun ? parentRun.depth + 1 : 0;
|
||||
const rootTaskRunId = parentRun?.rootTaskRunId ?? parentRun?.id;
|
||||
|
||||
// Resolve queue properties (same as triggerTask) so span presenter can find TaskQueue.
|
||||
// Best-effort: if resolution throws (e.g. request shape, missing worker), we still create
|
||||
// the run without queue/lockedQueueId so run creation and trace events never regress.
|
||||
let queueName: string | undefined;
|
||||
let lockedQueueId: string | undefined;
|
||||
try {
|
||||
const queueConcern = new DefaultQueueManager(this.prisma, this.engine, this.replicaPrisma);
|
||||
const bodyOptions = request.options as TriggerTaskRequest["body"]["options"];
|
||||
const triggerRequest: TriggerTaskRequest = {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: request.environment,
|
||||
body: {
|
||||
payload:
|
||||
typeof request.payload === "string"
|
||||
? request.payload
|
||||
: JSON.stringify(request.payload ?? {}),
|
||||
options: bodyOptions,
|
||||
},
|
||||
};
|
||||
|
||||
// Resolve the locked background worker if lockToVersion is set (same as triggerTask).
|
||||
// resolveQueueProperties requires the worker to be passed when lockToVersion is present.
|
||||
const lockedToBackgroundWorker = bodyOptions?.lockToVersion
|
||||
? await this.prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
projectId: request.environment.projectId,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
version: bodyOptions.lockToVersion,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
version: true,
|
||||
sdkVersion: true,
|
||||
cliVersion: true,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const resolved = await queueConcern.resolveQueueProperties(
|
||||
triggerRequest,
|
||||
lockedToBackgroundWorker ?? undefined
|
||||
);
|
||||
queueName = resolved.queueName;
|
||||
lockedQueueId = resolved.lockedQueueId;
|
||||
} catch (queueResolveError) {
|
||||
const err =
|
||||
queueResolveError instanceof Error
|
||||
? queueResolveError
|
||||
: new Error(String(queueResolveError));
|
||||
logger.warn("TriggerFailedTaskService: queue resolution failed, using defaults", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRunFriendlyId,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
|
||||
// Create the failed run inside a trace event span so it shows up in run logs
|
||||
const failedRun: TaskRun = await repository.traceEvent(
|
||||
request.taskId,
|
||||
{
|
||||
context: request.traceContext,
|
||||
spanParentAsLink: request.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: {
|
||||
id: request.environment.id,
|
||||
type: request.environment.type,
|
||||
organizationId: request.environment.organizationId,
|
||||
projectId: request.environment.projectId,
|
||||
project: { externalRef: request.environment.project.externalRef },
|
||||
},
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {},
|
||||
style: { icon: "task" },
|
||||
},
|
||||
incomplete: false,
|
||||
isError: true,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext) => {
|
||||
event.setAttribute("runId", failedRunFriendlyId);
|
||||
event.failWithError(taskRunError);
|
||||
|
||||
// `emitRunFailedEvent: false` because this call site owns the
|
||||
// trace-event lifecycle via the outer `traceEvent({
|
||||
// incomplete: false, isError: true })`. Letting the engine
|
||||
// emit `runFailed` here would race the
|
||||
// `completeFailedRunEvent` listener against the outer trace
|
||||
// event's own completion write for the same (traceId, spanId).
|
||||
// We re-trigger the alerts side directly after the trace
|
||||
// event closes, below.
|
||||
return await this.engine.createFailedTaskRun({
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: {
|
||||
id: request.environment.id,
|
||||
type: request.environment.type,
|
||||
project: { id: request.environment.project.id },
|
||||
organization: { id: request.environment.organization.id },
|
||||
},
|
||||
taskIdentifier: request.taskId,
|
||||
payload:
|
||||
typeof request.payload === "string"
|
||||
? request.payload
|
||||
: JSON.stringify(request.payload ?? ""),
|
||||
payloadType: request.payloadType ?? "application/json",
|
||||
error: taskRunError,
|
||||
parentTaskRunId: parentRun?.id,
|
||||
rootTaskRunId,
|
||||
depth,
|
||||
resumeParentOnCompletion: request.resumeParentOnCompletion,
|
||||
batch: request.batch,
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext: traceContext as Record<string, unknown>,
|
||||
taskEventStore: store,
|
||||
emitRunFailedEvent: false,
|
||||
...(queueName !== undefined && { queue: queueName }),
|
||||
...(lockedQueueId !== undefined && { lockedQueueId }),
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
// Alerts side of `runFailed` — the engine emit was suppressed
|
||||
// above so the trace-event completion isn't double-written; we
|
||||
// still need the alert pipeline to fire so customers' ERROR
|
||||
// channels see the failure. Best-effort: a failed enqueue logs
|
||||
// but doesn't block returning the friendlyId, mirroring the
|
||||
// engine handler's behaviour at runEngineHandlers.server.ts:81.
|
||||
try {
|
||||
await PerformTaskRunAlertsService.enqueue(failedRun.id);
|
||||
} catch (alertsError) {
|
||||
logger.warn("TriggerFailedTaskService: alert enqueue failed", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRun.friendlyId,
|
||||
error: alertsError instanceof Error ? alertsError.message : String(alertsError),
|
||||
});
|
||||
}
|
||||
|
||||
return failedRun.friendlyId;
|
||||
} catch (createError) {
|
||||
const createErrorMsg =
|
||||
createError instanceof Error ? createError.message : String(createError);
|
||||
logger.error("TriggerFailedTaskService: failed to create pre-failed TaskRun", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: mintedFriendlyId,
|
||||
originalError: request.errorMessage,
|
||||
createError: createErrorMsg,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a pre-failed run without trace events.
|
||||
* Used when the environment can't be fully resolved (e.g. environment not found)
|
||||
* and we can't create trace events or look up parent runs.
|
||||
*/
|
||||
async callWithoutTraceEvents(opts: {
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
taskId: string;
|
||||
payload: unknown;
|
||||
payloadType?: string;
|
||||
errorMessage: string;
|
||||
parentRunId?: string;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
batch?: { id: string; index: number };
|
||||
errorCode?: TaskRunErrorCodes;
|
||||
/** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
|
||||
runFriendlyId?: string;
|
||||
}): Promise<string | null> {
|
||||
// Held for the catch's log line; the in-try `const` is what consumers use.
|
||||
let mintedFriendlyId: string | undefined;
|
||||
|
||||
try {
|
||||
// Mint inside the try: classifying a user-supplied parentRunId throws on
|
||||
// an unclassifiable id, so keep it within the catch's null-return contract.
|
||||
const failedRunFriendlyId = await this.mintFailedRunFriendlyId({
|
||||
organizationId: opts.organizationId,
|
||||
environmentId: opts.environmentId,
|
||||
// No loaded org flags in this path; resolveRunIdMintKind falls back to a
|
||||
// single replica lookup by organizationId only when there is no parent.
|
||||
orgFeatureFlags: undefined,
|
||||
parentRunFriendlyId: opts.parentRunId,
|
||||
runFriendlyId: opts.runFriendlyId,
|
||||
});
|
||||
mintedFriendlyId = failedRunFriendlyId;
|
||||
|
||||
// Best-effort parent run lookup for rootTaskRunId/depth
|
||||
let parentTaskRunId: string | undefined;
|
||||
let rootTaskRunId: string | undefined;
|
||||
let depth = 0;
|
||||
|
||||
if (opts.parentRunId) {
|
||||
const parentRun = await this.runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(opts.parentRunId),
|
||||
runtimeEnvironmentId: opts.environmentId,
|
||||
},
|
||||
this.prisma
|
||||
);
|
||||
|
||||
if (parentRun) {
|
||||
parentTaskRunId = parentRun.id;
|
||||
rootTaskRunId = parentRun.rootTaskRunId ?? parentRun.id;
|
||||
depth = parentRun.depth + 1;
|
||||
} else {
|
||||
parentTaskRunId = RunId.fromFriendlyId(opts.parentRunId);
|
||||
}
|
||||
}
|
||||
|
||||
const failedRun = await this.engine.createFailedTaskRun({
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: {
|
||||
id: opts.environmentId,
|
||||
type: opts.environmentType,
|
||||
project: { id: opts.projectId },
|
||||
organization: { id: opts.organizationId },
|
||||
},
|
||||
taskIdentifier: opts.taskId,
|
||||
payload:
|
||||
typeof opts.payload === "string" ? opts.payload : JSON.stringify(opts.payload ?? ""),
|
||||
payloadType: opts.payloadType ?? "application/json",
|
||||
error: {
|
||||
type: "INTERNAL_ERROR" as const,
|
||||
code: opts.errorCode ?? TaskRunErrorCodes.UNSPECIFIED_ERROR,
|
||||
message: opts.errorMessage,
|
||||
},
|
||||
parentTaskRunId,
|
||||
rootTaskRunId,
|
||||
depth,
|
||||
resumeParentOnCompletion: opts.resumeParentOnCompletion,
|
||||
batch: opts.batch,
|
||||
// Suppress the engine's `runFailed` bus emit — the listener
|
||||
// (`runEngineHandlers.server.ts` `runFailed`) calls
|
||||
// `completeFailedRunEvent`, which writes a ClickHouse trace event
|
||||
// row keyed on (traceId, spanId). This caller has no trace
|
||||
// context (the method name is literally `callWithoutTraceEvents`)
|
||||
// so the emit would write a row with empty traceId/spanId —
|
||||
// orphan event in the store. We still want alert coverage,
|
||||
// though, so enqueue directly below.
|
||||
emitRunFailedEvent: false,
|
||||
});
|
||||
|
||||
// Alerts side of `runFailed` — the engine emit was suppressed
|
||||
// above so we don't create an orphan trace event; enqueue the
|
||||
// alert directly so customers' ERROR channels still see the
|
||||
// failure. Best-effort, mirroring the `call()` path.
|
||||
try {
|
||||
await PerformTaskRunAlertsService.enqueue(failedRun.id);
|
||||
} catch (alertsError) {
|
||||
logger.warn("TriggerFailedTaskService.callWithoutTraceEvents: alert enqueue failed", {
|
||||
taskId: opts.taskId,
|
||||
friendlyId: failedRun.friendlyId,
|
||||
error: alertsError instanceof Error ? alertsError.message : String(alertsError),
|
||||
});
|
||||
}
|
||||
|
||||
return failedRunFriendlyId;
|
||||
} catch (createError) {
|
||||
logger.error("TriggerFailedTaskService: failed to create pre-failed TaskRun (no trace)", {
|
||||
taskId: opts.taskId,
|
||||
friendlyId: mintedFriendlyId,
|
||||
originalError: opts.errorMessage,
|
||||
createError: createError instanceof Error ? createError.message : String(createError),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,825 @@
|
||||
import { describe, expect, vi } from "vitest";
|
||||
|
||||
// Mock the db prisma client. The service is constructed against a real
|
||||
// testcontainer prisma instead — these empty singletons only satisfy the
|
||||
// module-level imports of the production wiring (infrastructure boundary).
|
||||
vi.mock("~/db.server", () => ({
|
||||
prisma: {},
|
||||
$replica: {},
|
||||
runOpsNewPrisma: {},
|
||||
runOpsLegacyPrisma: {},
|
||||
runOpsNewReplica: {},
|
||||
runOpsLegacyReplica: {},
|
||||
}));
|
||||
// Inherited harness boilerplate. The parent read under test takes the
|
||||
// findRun(where, client) overload with this.prisma, so it does not consult this
|
||||
// flag; the mock only satisfies other wiring imported transitively.
|
||||
vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
|
||||
|
||||
vi.mock("~/services/platform.v3.server", async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as Record<string, unknown>;
|
||||
return {
|
||||
...actual,
|
||||
getEntitlement: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { RunEngine } from "@internal/run-engine";
|
||||
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests";
|
||||
import { assertNonNullable, containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import type { IOPacket } from "@trigger.dev/core/v3";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
|
||||
import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
|
||||
import type {
|
||||
EntitlementValidationParams,
|
||||
MaxAttemptsValidationParams,
|
||||
ParentRunValidationParams,
|
||||
PayloadProcessor,
|
||||
TagValidationParams,
|
||||
TracedEventSpan,
|
||||
TraceEventConcern,
|
||||
TriggerTaskRequest,
|
||||
TriggerTaskValidator,
|
||||
ValidationResult,
|
||||
} from "~/runEngine/types";
|
||||
import { RunEngineTriggerTaskService } from "./triggerTask.server";
|
||||
|
||||
vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout
|
||||
|
||||
class MockPayloadProcessor implements PayloadProcessor {
|
||||
async process(request: TriggerTaskRequest): Promise<IOPacket> {
|
||||
return {
|
||||
data: JSON.stringify(request.body.payload),
|
||||
dataType: "application/json",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Captures the `parentRun` the service resolved (via runStore.findRun) and
|
||||
// passed into validation, so a test can assert on the resolved parent without
|
||||
// mocking the read itself. Returns ok so the child triggers regardless.
|
||||
class CapturingParentRunValidator implements TriggerTaskValidator {
|
||||
public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset";
|
||||
|
||||
validateTags(_params: TagValidationParams): ValidationResult {
|
||||
return { ok: true };
|
||||
}
|
||||
validateEntitlement(_params: EntitlementValidationParams): Promise<ValidationResult> {
|
||||
return Promise.resolve({ ok: true });
|
||||
}
|
||||
validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult {
|
||||
return { ok: true };
|
||||
}
|
||||
validateParentRun(params: ParentRunValidationParams): ValidationResult {
|
||||
this.capturedParentRun = params.parentRun;
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
|
||||
class MockTraceEventConcern implements TraceEventConcern {
|
||||
async traceRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
|
||||
async traceIdempotentRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
_options: {
|
||||
existingRun: TaskRun;
|
||||
idempotencyKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
|
||||
async traceDebouncedRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
_options: {
|
||||
existingRun: TaskRun;
|
||||
debounceKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function buildEngine(prisma: any, redisOptions: any) {
|
||||
return new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0005,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
}
|
||||
|
||||
describe("RunEngineTriggerTaskService parent + locked-worker reads", () => {
|
||||
containerTest(
|
||||
"resolves the parent run through the run-ops store by minted run id",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger a ROOT run first to create a real parent TaskRun.
|
||||
const parentResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// Trigger a CHILD pointing at the parent's friendlyId. The service must
|
||||
// resolve the parent via runStore.findRun (minted RunId, env-scoped).
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: { parentRunId: parentResult.run.friendlyId },
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
// The capturing validator observed the resolved parent — proving the
|
||||
// read ran (against the container DB) and returned the right row.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
const capturedParent = validator.capturedParentRun;
|
||||
assertNonNullable(capturedParent);
|
||||
expect(capturedParent.id).toBe(parentResult.run.id);
|
||||
expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId);
|
||||
|
||||
// depth and root carry through — proving parentRun.depth and the parent
|
||||
// id were read off the resolved row and threaded into the child.
|
||||
const parentRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: parentResult.run.id },
|
||||
});
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
|
||||
expect(childRow.depth).toBe(parentRow.depth + 1);
|
||||
expect(childRow.parentTaskRunId).toBe(parentRow.id);
|
||||
expect(childRow.rootTaskRunId).toBe(parentRow.id);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"scopes the parent lookup to the run's environment (cross-env parent is not resolved)",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
// Two independent authenticated environments. The setup helper hardcodes
|
||||
// several globally-unique fields (org/project slug, env apiKey/pkApiKey,
|
||||
// worker-group token hash), so rename envA's before the second call to
|
||||
// avoid unique-constraint collisions.
|
||||
const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
await prisma.organization.update({
|
||||
where: { id: envA.organizationId },
|
||||
data: { slug: `${envA.organization.slug}-a` },
|
||||
});
|
||||
await prisma.project.update({
|
||||
where: { id: envA.projectId },
|
||||
data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` },
|
||||
});
|
||||
await prisma.runtimeEnvironment.update({
|
||||
where: { id: envA.id },
|
||||
data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` },
|
||||
});
|
||||
await prisma.workerGroupToken.updateMany({
|
||||
where: { tokenHash: "token_hash" },
|
||||
data: { tokenHash: "token_hash_a" },
|
||||
});
|
||||
await prisma.workerInstanceGroup.updateMany({
|
||||
where: { masterQueue: "default" },
|
||||
data: { masterQueue: "default_a" },
|
||||
});
|
||||
const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
expect(envA.id).not.toBe(envB.id);
|
||||
expect(envA.organizationId).not.toBe(envB.organizationId);
|
||||
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, envA, taskIdentifier);
|
||||
await setupBackgroundWorker(engine, envB, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// A real parent run in envA.
|
||||
const parentResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envA,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// Trigger a child in envB pointing at the envA parent's friendlyId. The
|
||||
// env guard in runStore.findRun's `where` rejects the cross-env parent
|
||||
// in a single query, so the resolved parentRun is null.
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envB,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: { parentRunId: parentResult.run.friendlyId },
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
// validateParentRun was called with no resolved parent.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
expect(validator.capturedParentRun ?? null).toBeNull();
|
||||
|
||||
// The child still triggered, at the root depth with no parent linkage —
|
||||
// confirming the cross-env parent was dropped, not silently joined.
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
expect(childRow.depth).toBe(0);
|
||||
expect(childRow.parentTaskRunId).toBeNull();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"resolves the locked background worker on the control-plane client with no cross-DB join",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
// Read the seeded worker row to get its real version/id.
|
||||
const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: worker.id },
|
||||
});
|
||||
|
||||
// Counting proxy over the control-plane client. `this.prisma` is ALWAYS
|
||||
// the control-plane client; the locked-worker lookup is a DIRECT
|
||||
// backgroundWorker.findFirst on it. The parent read uses a DIFFERENT
|
||||
// call (runStore.findRun → taskRun), so a single call() issues two
|
||||
// separate single-table reads — never one cross-seam join. Here we count
|
||||
// the findFirst calls and capture their args to assert no include/join.
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
// The queue manager gets the real (unproxied) prisma so the counting
|
||||
// proxy only observes reads issued by the service itself.
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: workerRow.version },
|
||||
},
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
// Observable proof the locked worker was resolved on the control-plane
|
||||
// client: the created run records the worker id in lockedToVersionId.
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
expect(runRow.lockedToVersionId).toBe(workerRow.id);
|
||||
expect(runRow.taskVersion).toBe(workerRow.version);
|
||||
|
||||
// Exactly one backgroundWorker.findFirst fired for the locked-worker read.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
|
||||
// NO-JOIN assertion: the read referenced ONLY the backgroundWorker table.
|
||||
// No `include` (which would join into another table); the `select` lists
|
||||
// only backgroundWorker scalar columns.
|
||||
const args = findFirstArgs[0];
|
||||
assertNonNullable(args);
|
||||
expect(args.include).toBeUndefined();
|
||||
expect(Object.keys(args.select ?? {}).sort()).toEqual([
|
||||
"cliVersion",
|
||||
"id",
|
||||
"sdkVersion",
|
||||
"version",
|
||||
]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"issues two independent single-table reads when one call supplies both parentRunId and lockToVersion",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: worker.id },
|
||||
});
|
||||
|
||||
// Count BOTH reads issued by the service on the control-plane client:
|
||||
// the parent read (runStore.findRun → taskRun.findFirst) and the
|
||||
// locked-worker read (backgroundWorker.findFirst). Capture every
|
||||
// findFirst arg so we can assert no read carries a cross-seam include.
|
||||
let taskRunFindFirstCalls = 0;
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
if (prop === "taskRun") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(trTarget, trProp, trReceiver) {
|
||||
if (trProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
taskRunFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(trTarget, trProp, trReceiver);
|
||||
return typeof value === "function" ? value.bind(trTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
// queueConcern/idempotency get the real unproxied prisma so the
|
||||
// counting proxy only observes reads issued by the service itself.
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// ROOT parent first (uses the unproxied prisma via a separate service so
|
||||
// its internal reads don't pollute the child's counts).
|
||||
const parentService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
const parentResult = await parentService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// CHILD supplying BOTH parentRunId AND lockToVersion in one call.
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: {
|
||||
parentRunId: parentResult.run.friendlyId,
|
||||
lockToVersion: workerRow.version,
|
||||
},
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
const parentRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: parentResult.run.id },
|
||||
});
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
|
||||
// Child resolved the parent (single-table parent read).
|
||||
expect(childRow.parentTaskRunId).toBe(parentRow.id);
|
||||
expect(childRow.depth).toBe(parentRow.depth + 1);
|
||||
|
||||
// Child locked to the worker (single-table worker read).
|
||||
expect(childRow.lockedToVersionId).toBe(workerRow.id);
|
||||
expect(childRow.taskVersion).toBe(workerRow.version);
|
||||
|
||||
// Exactly one backgroundWorker.findFirst fired for the locked-worker read,
|
||||
// and at least one taskRun.findFirst fired for the parent read.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// NO-JOIN proof: no captured read carried an `include` joining
|
||||
// taskRun <-> backgroundWorker. Every findFirst arg has include undefined.
|
||||
for (const args of findFirstArgs) {
|
||||
expect(args?.include).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"lockToVersion matching no worker rejects the trigger after a single scalar-only worker read",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
const bogusVersion = "v-does-not-exist-0000";
|
||||
// The no-match worker read returns null; the queue concern then rejects
|
||||
// the trigger rather than silently locking the run to a phantom version.
|
||||
await expect(
|
||||
triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: bogusVersion },
|
||||
},
|
||||
})
|
||||
).rejects.toThrow(/no worker found with that version/);
|
||||
|
||||
// No run was locked to the bogus version (none was created).
|
||||
const lockedRuns = await prisma.taskRun.findMany({
|
||||
where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion },
|
||||
});
|
||||
expect(lockedRuns).toEqual([]);
|
||||
|
||||
// The lone worker read fired exactly once with the scalar-only select and
|
||||
// no cross-seam include.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
const args = findFirstArgs[0];
|
||||
assertNonNullable(args);
|
||||
expect(args.include).toBeUndefined();
|
||||
expect(Object.keys(args.select ?? {}).sort()).toEqual([
|
||||
"cliVersion",
|
||||
"id",
|
||||
"sdkVersion",
|
||||
"version",
|
||||
]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"does not resolve a locked worker from a different environment",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
// Two independent authenticated environments. Rename envA's globally-unique
|
||||
// fields before the second setup call to avoid unique-constraint collisions.
|
||||
const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
await prisma.organization.update({
|
||||
where: { id: envA.organizationId },
|
||||
data: { slug: `${envA.organization.slug}-a` },
|
||||
});
|
||||
await prisma.project.update({
|
||||
where: { id: envA.projectId },
|
||||
data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` },
|
||||
});
|
||||
await prisma.runtimeEnvironment.update({
|
||||
where: { id: envA.id },
|
||||
data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` },
|
||||
});
|
||||
await prisma.workerGroupToken.updateMany({
|
||||
where: { tokenHash: "token_hash" },
|
||||
data: { tokenHash: "token_hash_a" },
|
||||
});
|
||||
await prisma.workerInstanceGroup.updateMany({
|
||||
where: { masterQueue: "default" },
|
||||
data: { masterQueue: "default_a" },
|
||||
});
|
||||
const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
expect(envA.id).not.toBe(envB.id);
|
||||
expect(envA.organizationId).not.toBe(envB.organizationId);
|
||||
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier);
|
||||
const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier);
|
||||
|
||||
const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: workerA.id },
|
||||
});
|
||||
const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: workerB.id },
|
||||
});
|
||||
// Both seeded workers share the same version string.
|
||||
expect(workerARow.version).toBe(workerBRow.version);
|
||||
expect(workerARow.id).not.toBe(workerBRow.id);
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger in envB locking to the shared version string.
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envB,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: workerBRow.version },
|
||||
},
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
// The projectId + runtimeEnvironmentId guard in the single-table worker
|
||||
// read resolves envB's worker, never envA's same-version worker.
|
||||
expect(runRow.lockedToVersionId).toBe(workerBRow.id);
|
||||
expect(runRow.lockedToVersionId).not.toBe(workerARow.id);
|
||||
expect(runRow.taskVersion).toBe(workerBRow.version);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger with NO parentRunId.
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "root" } },
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
// The validator ran but received no resolved parent: the parent read was
|
||||
// skipped because no parentRunId was supplied.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
expect(validator.capturedParentRun).toBeUndefined();
|
||||
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
expect(runRow.depth).toBe(0);
|
||||
expect(runRow.parentTaskRunId).toBeNull();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,942 @@
|
||||
import {
|
||||
type RunEngine,
|
||||
RunDuplicateIdempotencyKeyError,
|
||||
RunOneTimeUseTokenError,
|
||||
} from "@internal/run-engine";
|
||||
import type { Tracer } from "@opentelemetry/api";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import {
|
||||
type TriggerTaskRequestBody,
|
||||
RunAnnotations,
|
||||
TaskRunError,
|
||||
taskRunErrorEnhancer,
|
||||
taskRunErrorToString,
|
||||
TriggerTraceContext,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import {
|
||||
parseTraceparent,
|
||||
RunId,
|
||||
serializeTraceparent,
|
||||
stringifyDuration,
|
||||
} from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { parseDelay } from "~/utils/delays";
|
||||
import { handleMetadataPacket } from "~/utils/packets";
|
||||
import { startSpan } from "~/v3/tracing.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
|
||||
import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
||||
import type {
|
||||
TriggerTaskServiceOptions,
|
||||
TriggerTaskServiceResult,
|
||||
} from "../../v3/services/triggerTask.server";
|
||||
import { clampMaxDuration } from "../../v3/utils/maxDuration";
|
||||
import {
|
||||
type IdempotencyKeyConcern,
|
||||
type ClaimedIdempotency,
|
||||
} from "../concerns/idempotencyKeys.server";
|
||||
import {
|
||||
resolveScheduledQueueSplitEnabled,
|
||||
workerQueueForRun,
|
||||
} from "../concerns/workerQueueSplit.server";
|
||||
import { resolveComputeMigration } from "../concerns/computeMigration.server";
|
||||
import { workerRegionRegistry, backingForQueue, regionForQueue } from "~/v3/workerRegions.server";
|
||||
import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
|
||||
import {
|
||||
publishClaim as publishMollifierClaim,
|
||||
releaseClaim as releaseMollifierClaim,
|
||||
} from "~/v3/mollifier/idempotencyClaim.server";
|
||||
import type {
|
||||
PayloadProcessor,
|
||||
QueueManager,
|
||||
TraceEventConcern,
|
||||
TriggerRacepoints,
|
||||
TriggerRacepointSystem,
|
||||
TriggerTaskRequest,
|
||||
TriggerTaskValidator,
|
||||
} from "../types";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
evaluateGate as defaultEvaluateGate,
|
||||
type GateOutcome,
|
||||
type MollifierEvaluateGate,
|
||||
} from "~/v3/mollifier/mollifierGate.server";
|
||||
import {
|
||||
getMollifierBuffer as defaultGetMollifierBuffer,
|
||||
type MollifierGetBuffer,
|
||||
} from "~/v3/mollifier/mollifierBuffer.server";
|
||||
import { mollifyTrigger } from "~/v3/mollifier/mollifierMollify.server";
|
||||
import { QueueSizeLimitExceededError, ServiceValidationError } from "~/v3/services/common.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
|
||||
class NoopTriggerRacepointSystem implements TriggerRacepointSystem {
|
||||
async waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void> {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export class RunEngineTriggerTaskService {
|
||||
private readonly queueConcern: QueueManager;
|
||||
private readonly validator: TriggerTaskValidator;
|
||||
private readonly payloadProcessor: PayloadProcessor;
|
||||
private readonly idempotencyKeyConcern: IdempotencyKeyConcern;
|
||||
private readonly prisma: PrismaClientOrTransaction;
|
||||
private readonly engine: RunEngine;
|
||||
private readonly tracer: Tracer;
|
||||
private readonly traceEventConcern: TraceEventConcern;
|
||||
private readonly triggerRacepointSystem: TriggerRacepointSystem;
|
||||
private readonly metadataMaximumSize: number;
|
||||
// Mollifier hooks are DI'd so tests can drive the call-site's mollify branch
|
||||
// deterministically (stub the gate to return mollify, inject a real or fake
|
||||
// buffer, force the global-enabled predicate to true so the call site
|
||||
// doesn't short-circuit on an unset env). In production all three default
|
||||
// to the live module-level singletons + env read.
|
||||
private readonly evaluateGate: MollifierEvaluateGate;
|
||||
private readonly getMollifierBuffer: MollifierGetBuffer;
|
||||
private readonly isMollifierGloballyEnabled: () => boolean;
|
||||
|
||||
constructor(opts: {
|
||||
prisma: PrismaClientOrTransaction;
|
||||
engine: RunEngine;
|
||||
queueConcern: QueueManager;
|
||||
validator: TriggerTaskValidator;
|
||||
payloadProcessor: PayloadProcessor;
|
||||
idempotencyKeyConcern: IdempotencyKeyConcern;
|
||||
traceEventConcern: TraceEventConcern;
|
||||
tracer: Tracer;
|
||||
metadataMaximumSize: number;
|
||||
triggerRacepointSystem?: TriggerRacepointSystem;
|
||||
evaluateGate?: MollifierEvaluateGate;
|
||||
getMollifierBuffer?: MollifierGetBuffer;
|
||||
isMollifierGloballyEnabled?: () => boolean;
|
||||
}) {
|
||||
this.prisma = opts.prisma;
|
||||
this.engine = opts.engine;
|
||||
this.queueConcern = opts.queueConcern;
|
||||
this.validator = opts.validator;
|
||||
this.payloadProcessor = opts.payloadProcessor;
|
||||
this.idempotencyKeyConcern = opts.idempotencyKeyConcern;
|
||||
this.tracer = opts.tracer;
|
||||
this.traceEventConcern = opts.traceEventConcern;
|
||||
this.metadataMaximumSize = opts.metadataMaximumSize;
|
||||
this.triggerRacepointSystem = opts.triggerRacepointSystem ?? new NoopTriggerRacepointSystem();
|
||||
this.evaluateGate = opts.evaluateGate ?? defaultEvaluateGate;
|
||||
this.getMollifierBuffer = opts.getMollifierBuffer ?? defaultGetMollifierBuffer;
|
||||
this.isMollifierGloballyEnabled =
|
||||
opts.isMollifierGloballyEnabled ?? (() => env.TRIGGER_MOLLIFIER_ENABLED === "1");
|
||||
}
|
||||
|
||||
// Mint a new run's friendlyId. The id-kind decides which store the run is born
|
||||
// in (cuid → legacy store, run-ops id → new store), so the whole subgraph of a run
|
||||
// must agree. Two cases:
|
||||
//
|
||||
// - ROOT run (no parent): mint by the environment's cutover setting.
|
||||
// - CHILD run (has a parent): inherit the parent's residency by id-shape, so a
|
||||
// parent and child never split across stores (run-ops parent → run-ops child,
|
||||
// cuid parent → cuid child).
|
||||
// `region` is the caller-requested region (body.options.region). The id is
|
||||
// minted before the worker queue is resolved (the idempotency concern needs
|
||||
// the friendlyId first), so the stamped region char reflects the requested
|
||||
// region — or the default char when the run targets the default region.
|
||||
private async mintRunFriendlyId(
|
||||
environment: AuthenticatedEnvironment,
|
||||
parentRunFriendlyId?: string,
|
||||
region?: string
|
||||
): Promise<string> {
|
||||
const mintKind = parentRunFriendlyId
|
||||
? resolveInheritedMintKind(parentRunFriendlyId)
|
||||
: await resolveRunIdMintKind({
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
});
|
||||
|
||||
return mintFriendlyIdForKind(mintKind, region);
|
||||
}
|
||||
|
||||
public async call({
|
||||
taskId,
|
||||
environment,
|
||||
body,
|
||||
options = {},
|
||||
attempt = 0,
|
||||
}: {
|
||||
taskId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
body: TriggerTaskRequestBody;
|
||||
options?: TriggerTaskServiceOptions;
|
||||
attempt?: number;
|
||||
}): Promise<TriggerTaskServiceResult | undefined> {
|
||||
// Pre-gate idempotency-claim ownership. Set inside the span when
|
||||
// `IdempotencyKeyConcern.handleTriggerRequest` returns `claim:
|
||||
// {...}`. The try/catch below resolves it once the span finishes.
|
||||
let idempotencyClaim: ClaimedIdempotency | undefined;
|
||||
try {
|
||||
const result = await startSpan(
|
||||
this.tracer,
|
||||
"RunEngineTriggerTaskService.call()",
|
||||
async (span) => {
|
||||
span.setAttribute("taskId", taskId);
|
||||
span.setAttribute("attempt", attempt);
|
||||
|
||||
// Mint the run id. A caller-supplied id (idempotent retry) wins;
|
||||
// otherwise mint by residency — inheriting the parent's store when a
|
||||
// parent is present, else the environment's setting.
|
||||
const runFriendlyId =
|
||||
options?.runFriendlyId ??
|
||||
(await this.mintRunFriendlyId(
|
||||
environment,
|
||||
body.options?.parentRunId,
|
||||
body.options?.region
|
||||
));
|
||||
const triggerRequest = {
|
||||
taskId,
|
||||
friendlyId: runFriendlyId,
|
||||
environment,
|
||||
body,
|
||||
options,
|
||||
} satisfies TriggerTaskRequest;
|
||||
|
||||
const maxAttemptsValidation = this.validator.validateMaxAttempts({
|
||||
taskId,
|
||||
attempt,
|
||||
});
|
||||
|
||||
if (!maxAttemptsValidation.ok) {
|
||||
throw maxAttemptsValidation.error;
|
||||
}
|
||||
|
||||
const tagValidation = this.validator.validateTags({
|
||||
tags: body.options?.tags,
|
||||
});
|
||||
|
||||
if (!tagValidation.ok) {
|
||||
throw tagValidation.error;
|
||||
}
|
||||
|
||||
let planType: string | undefined;
|
||||
|
||||
if (!options.skipChecks) {
|
||||
const entitlementValidation = await this.validator.validateEntitlement({
|
||||
environment,
|
||||
});
|
||||
|
||||
if (!entitlementValidation.ok) {
|
||||
throw entitlementValidation.error;
|
||||
}
|
||||
|
||||
planType = entitlementValidation.plan?.type;
|
||||
} else {
|
||||
// When skipChecks is enabled, planType should be passed via options
|
||||
planType = options.planType;
|
||||
|
||||
if (!planType) {
|
||||
logger.warn("Plan type not set but skipChecks is enabled", {
|
||||
taskId,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
type: environment.type,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Parse delay from either explicit delay option or debounce.delay
|
||||
const delaySource = body.options?.delay ?? body.options?.debounce?.delay;
|
||||
const [parseDelayError, delayUntil] = await tryCatch(parseDelay(delaySource));
|
||||
|
||||
if (parseDelayError) {
|
||||
throw new ServiceValidationError(`Invalid delay ${delaySource}`);
|
||||
}
|
||||
|
||||
// Validate debounce options
|
||||
if (body.options?.debounce) {
|
||||
if (!delayUntil) {
|
||||
throw new ServiceValidationError(
|
||||
`Debounce requires a valid delay duration. Provided: ${body.options.debounce.delay}`
|
||||
);
|
||||
}
|
||||
|
||||
// Always validate debounce.delay separately since it's used for rescheduling
|
||||
// This catches the case where options.delay is valid but debounce.delay is invalid
|
||||
const [debounceDelayError, debounceDelayUntil] = await tryCatch(
|
||||
parseDelay(body.options.debounce.delay)
|
||||
);
|
||||
|
||||
if (debounceDelayError || !debounceDelayUntil) {
|
||||
throw new ServiceValidationError(
|
||||
`Invalid debounce delay: ${body.options.debounce.delay}. ` +
|
||||
`Supported formats: {number}s, {number}m, {number}h, {number}d, {number}w`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const parentRun = body.options?.parentRunId
|
||||
? await runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(body.options.parentRunId),
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
this.prisma
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const parentRunValidation = this.validator.validateParentRun({
|
||||
taskId,
|
||||
parentRun: parentRun ?? undefined,
|
||||
resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
if (!parentRunValidation.ok) {
|
||||
throw parentRunValidation.error;
|
||||
}
|
||||
|
||||
const idempotencyKeyConcernResult = await this.idempotencyKeyConcern.handleTriggerRequest(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore
|
||||
);
|
||||
|
||||
if (idempotencyKeyConcernResult.isCached) {
|
||||
return idempotencyKeyConcernResult;
|
||||
}
|
||||
|
||||
const {
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
claim: claimResult,
|
||||
} = idempotencyKeyConcernResult;
|
||||
|
||||
// If we own an idempotency claim, the trigger pipeline below MUST
|
||||
// resolve it — publish on success so waiters see our runId,
|
||||
// release on error so the next claimant can retry. Stored in an
|
||||
// outer scope so the try/catch at the bottom of `callV2` can act
|
||||
// on whichever return path or throw the pipeline takes.
|
||||
idempotencyClaim = claimResult;
|
||||
|
||||
if (idempotencyKey) {
|
||||
await this.triggerRacepointSystem.waitForRacepoint({
|
||||
racepoint: "idempotencyKey",
|
||||
id: idempotencyKey,
|
||||
});
|
||||
}
|
||||
|
||||
const lockedToBackgroundWorker = body.options?.lockToVersion
|
||||
? await this.prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
projectId: environment.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
version: body.options?.lockToVersion,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
version: true,
|
||||
sdkVersion: true,
|
||||
cliVersion: true,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const { queueName, lockedQueueId, taskTtl, taskKind } =
|
||||
await this.queueConcern.resolveQueueProperties(
|
||||
triggerRequest,
|
||||
lockedToBackgroundWorker ?? undefined
|
||||
);
|
||||
|
||||
// Resolve TTL with precedence: per-trigger > task-level > dev default
|
||||
let ttl: string | undefined;
|
||||
|
||||
if (body.options?.ttl !== undefined) {
|
||||
ttl =
|
||||
typeof body.options.ttl === "number"
|
||||
? stringifyDuration(body.options.ttl)
|
||||
: body.options.ttl;
|
||||
} else {
|
||||
ttl = taskTtl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined);
|
||||
}
|
||||
|
||||
if (!options.skipChecks) {
|
||||
const queueSizeGuard = await this.queueConcern.validateQueueLimits(
|
||||
environment,
|
||||
queueName
|
||||
);
|
||||
|
||||
if (!queueSizeGuard.ok) {
|
||||
throw new QueueSizeLimitExceededError(
|
||||
`Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}`,
|
||||
queueSizeGuard.maximumSize ?? 0,
|
||||
undefined,
|
||||
"warn"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataPacket = body.options?.metadata
|
||||
? handleMetadataPacket(
|
||||
body.options?.metadata,
|
||||
body.options?.metadataType ?? "application/json",
|
||||
this.metadataMaximumSize
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const tags = (
|
||||
body.options?.tags
|
||||
? typeof body.options.tags === "string"
|
||||
? [body.options.tags]
|
||||
: body.options.tags
|
||||
: []
|
||||
).filter((tag) => tag.trim().length > 0);
|
||||
|
||||
const depth = parentRun ? parentRun.depth + 1 : 0;
|
||||
|
||||
const workerQueueResult = await this.queueConcern.getWorkerQueue(
|
||||
environment,
|
||||
body.options?.region
|
||||
);
|
||||
const baseWorkerQueue = workerQueueResult?.masterQueue;
|
||||
const enableFastPath = workerQueueResult?.enableFastPath ?? false;
|
||||
|
||||
// Rewrite the region to its compute backing for migration-enrolled orgs,
|
||||
// from the in-memory snapshots (no DB query). A cold read (registry not yet
|
||||
// loaded) returns undefined/[] and the resolver falls back to not-migrated.
|
||||
const workerGroups = workerRegionRegistry.current() ?? [];
|
||||
const region = baseWorkerQueue
|
||||
? regionForQueue(baseWorkerQueue, workerGroups)
|
||||
: undefined;
|
||||
const backing = baseWorkerQueue
|
||||
? backingForQueue(baseWorkerQueue, workerGroups)
|
||||
: undefined;
|
||||
const migrated = resolveComputeMigration({
|
||||
baseWorkerQueue,
|
||||
baseEnableFastPath: enableFastPath,
|
||||
region,
|
||||
backing,
|
||||
planType,
|
||||
orgId: environment.organization.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null,
|
||||
flags: globalFlagsRegistry.current(),
|
||||
envType: environment.type,
|
||||
});
|
||||
|
||||
const triggerSource = options.triggerSource ?? "api";
|
||||
const triggerAction = options.triggerAction ?? "trigger";
|
||||
const parentAnnotations = RunAnnotations.safeParse(parentRun?.annotations).data;
|
||||
const annotations = {
|
||||
triggerSource,
|
||||
triggerAction,
|
||||
rootTriggerSource: parentAnnotations?.rootTriggerSource ?? triggerSource,
|
||||
rootScheduleId: parentAnnotations?.rootScheduleId || options.scheduleId || undefined,
|
||||
taskKind: taskKind ?? "STANDARD",
|
||||
};
|
||||
|
||||
// Route runs in a scheduled lineage (the scheduled run itself and every
|
||||
// descendant, via the propagated rootTriggerSource) to a dedicated
|
||||
// `<region>:scheduled` worker queue so a separate consumer fleet can
|
||||
// dequeue them independently of standard/agent runs. Gated per-org with
|
||||
// a global default, never applied to dev. Reads only the in-memory org
|
||||
// flags already on the environment — no DB query on the hot path.
|
||||
const scheduledQueueSplitEnabled =
|
||||
environment.type !== "DEVELOPMENT" &&
|
||||
resolveScheduledQueueSplitEnabled({
|
||||
orgFeatureFlags: environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null,
|
||||
globalDefault: env.TRIGGER_WORKER_QUEUE_SCHEDULED_SPLIT_ENABLED === "1",
|
||||
});
|
||||
const workerQueue =
|
||||
migrated.workerQueue !== undefined
|
||||
? workerQueueForRun({
|
||||
workerQueue: migrated.workerQueue,
|
||||
rootTriggerSource: annotations.rootTriggerSource,
|
||||
splitEnabled: scheduledQueueSplitEnabled,
|
||||
})
|
||||
: migrated.workerQueue;
|
||||
|
||||
try {
|
||||
return await this.traceEventConcern.traceRun(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore,
|
||||
async (event, store) => {
|
||||
event.setAttribute("queueName", queueName);
|
||||
span.setAttribute("queueName", queueName);
|
||||
event.setAttribute("runId", runFriendlyId);
|
||||
span.setAttribute("runId", runFriendlyId);
|
||||
|
||||
// Short-circuit when mollifier is globally off (the default
|
||||
// for every deployment that hasn't opted in). Avoids the
|
||||
// GateInputs allocation, the deps spread inside `evaluateGate`,
|
||||
// and the `mollifier.decisions{outcome=pass_through}` OTel
|
||||
// increment on every trigger — `triggerTask` is the
|
||||
// highest-throughput code path in the system. The check goes
|
||||
// through a DI'd predicate so unit tests that inject a custom
|
||||
// `evaluateGate` can also override the gate-on check (the
|
||||
// default reads `env.TRIGGER_MOLLIFIER_ENABLED`, which is "0"
|
||||
// in CI where no .env file is present).
|
||||
//
|
||||
// Batch items bypass the mollifier gate entirely.
|
||||
//
|
||||
// The mollify path returns a stripped run-shape `{ id,
|
||||
// friendlyId, spanId }` with no PG row written. Batch
|
||||
// tracking relies on `BatchTaskRunItem`, a join row whose
|
||||
// `taskRunId` column has a NOT NULL FK to `TaskRun.id` —
|
||||
// creating that join at trigger-time (in
|
||||
// `batchTriggerV3.server.ts:871`) fails with FK violation
|
||||
// for any mollified item, and skipping it at trigger-time
|
||||
// would silently drop the batch↔run link forever because
|
||||
// the drainer's materialise path doesn't (yet) create
|
||||
// `BatchTaskRunItem`. Either side alone is wrong:
|
||||
// - skip at trigger-time only → batch progress
|
||||
// under-reports forever, `batchTriggerAndWait` parent
|
||||
// stays parked
|
||||
// - mollify at trigger-time only → FK violation, 500
|
||||
//
|
||||
// The proper end state is a drainer-side
|
||||
// `BatchTaskRunItem` create-on-materialise (the snapshot
|
||||
// already carries `batch: { id, index }` so the drainer
|
||||
// has the info). That belongs in the drainer / replay PR,
|
||||
// not here. Until that lands, batch triggers pass-through
|
||||
// — they lose the burst-protection benefit, but the path
|
||||
// works end-to-end.
|
||||
const skipMollifierForBatch = !!options.batchId;
|
||||
const mollifierOutcome: GateOutcome | null =
|
||||
this.isMollifierGloballyEnabled() && !skipMollifierForBatch
|
||||
? await this.evaluateGate({
|
||||
envId: environment.id,
|
||||
orgId: environment.organizationId,
|
||||
taskId,
|
||||
orgFeatureFlags:
|
||||
(environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null) ?? null,
|
||||
options: {
|
||||
debounce: body.options?.debounce,
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
parentTaskRunId: body.options?.parentRunId,
|
||||
resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
|
||||
},
|
||||
})
|
||||
: null;
|
||||
|
||||
// When the gate says mollify, write the engine.trigger input
|
||||
// snapshot into the Redis buffer and return a synthesised
|
||||
// TriggerTaskServiceResult. The customer never waits on
|
||||
// Postgres; the drainer materialises the run later by replaying
|
||||
// engine.trigger against the snapshot. The run span has already
|
||||
// been opened by traceRun above (PARTIAL event in ClickHouse),
|
||||
// so its traceId/spanId live in the snapshot and the drainer's
|
||||
// `mollifier.drained` span parents on the same trace — buffered
|
||||
// runs become visible in the dashboard's trace view immediately,
|
||||
// not only after the drainer fires.
|
||||
if (mollifierOutcome?.action === "mollify") {
|
||||
const mollifierBuffer = this.getMollifierBuffer();
|
||||
if (mollifierBuffer && !body.options?.debounce) {
|
||||
event.setAttribute("mollifier.reason", mollifierOutcome.decision.reason);
|
||||
event.setAttribute("mollifier.count", String(mollifierOutcome.decision.count));
|
||||
event.setAttribute(
|
||||
"mollifier.threshold",
|
||||
String(mollifierOutcome.decision.threshold)
|
||||
);
|
||||
event.setAttribute("taskRunId", runFriendlyId);
|
||||
|
||||
const payloadPacket = await this.payloadProcessor.process(triggerRequest);
|
||||
|
||||
const engineTriggerInput = this.#buildEngineTriggerInput({
|
||||
runFriendlyId,
|
||||
environment,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
body,
|
||||
options,
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
workerQueue,
|
||||
region: migrated.region,
|
||||
enableFastPath: migrated.enableFastPath,
|
||||
lockedToBackgroundWorker: lockedToBackgroundWorker ?? undefined,
|
||||
delayUntil,
|
||||
ttl,
|
||||
metadataPacket,
|
||||
tags,
|
||||
depth,
|
||||
parentRun: parentRun ?? undefined,
|
||||
annotations,
|
||||
planType,
|
||||
taskId,
|
||||
payloadPacket,
|
||||
traceContext: this.#propagateExternalTraceContext(
|
||||
event.traceContext,
|
||||
parentRun?.traceContext,
|
||||
event.traceparent?.spanId
|
||||
),
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
parentSpanId:
|
||||
options.parentAsLinkType === "replay"
|
||||
? undefined
|
||||
: event.traceparent?.spanId,
|
||||
taskEventStore: store,
|
||||
});
|
||||
|
||||
const result = await mollifyTrigger({
|
||||
runFriendlyId,
|
||||
environmentId: environment.id,
|
||||
organizationId: environment.organizationId,
|
||||
engineTriggerInput,
|
||||
decision: mollifierOutcome.decision,
|
||||
buffer: mollifierBuffer,
|
||||
// Idempotency-key triple wires the buffer's SETNX into
|
||||
// the trigger-time dedup symmetric with PG.
|
||||
idempotencyKey,
|
||||
taskIdentifier: taskId,
|
||||
});
|
||||
|
||||
logger.debug("mollifier.buffered", {
|
||||
runId: runFriendlyId,
|
||||
envId: environment.id,
|
||||
orgId: environment.organizationId,
|
||||
taskId,
|
||||
reason: mollifierOutcome.decision.reason,
|
||||
});
|
||||
|
||||
// Synthetic result is structurally narrower than the full
|
||||
// TaskRun; the route handler only reads
|
||||
// `result.run.friendlyId`. traceRun flushes the PARTIAL
|
||||
// run-span event to ClickHouse on callback return.
|
||||
// `isMollified` flags the route to skip the request-
|
||||
// idempotency cache write — see the field's contract on
|
||||
// `TriggerTaskServiceResult`.
|
||||
return {
|
||||
...(result as unknown as TriggerTaskServiceResult),
|
||||
isMollified: true,
|
||||
};
|
||||
}
|
||||
if (!mollifierBuffer) {
|
||||
logger.warn(
|
||||
"mollifier gate said mollify but buffer is null — falling through to pass-through"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const payloadPacket = await this.payloadProcessor.process(triggerRequest);
|
||||
|
||||
const baseEngineInput = this.#buildEngineTriggerInput({
|
||||
runFriendlyId,
|
||||
environment,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
body,
|
||||
options,
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
workerQueue,
|
||||
region: migrated.region,
|
||||
enableFastPath: migrated.enableFastPath,
|
||||
lockedToBackgroundWorker: lockedToBackgroundWorker ?? undefined,
|
||||
delayUntil,
|
||||
ttl,
|
||||
metadataPacket,
|
||||
tags,
|
||||
depth,
|
||||
parentRun: parentRun ?? undefined,
|
||||
annotations,
|
||||
planType,
|
||||
taskId,
|
||||
payloadPacket,
|
||||
traceContext: this.#propagateExternalTraceContext(
|
||||
event.traceContext,
|
||||
parentRun?.traceContext,
|
||||
event.traceparent?.spanId
|
||||
),
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
parentSpanId:
|
||||
options.parentAsLinkType === "replay" ? undefined : event.traceparent?.spanId,
|
||||
taskEventStore: store,
|
||||
});
|
||||
|
||||
const taskRun = await this.engine.trigger(
|
||||
{
|
||||
...baseEngineInput,
|
||||
// onDebounced is a closure over webapp state (triggerRequest +
|
||||
// traceEventConcern) and can't be serialised into the mollifier
|
||||
// snapshot. The pass-through path attaches it here; the drainer
|
||||
// path replays without it. The debounce and triggerAndWait gate
|
||||
// bypasses ensure neither reaches the mollify branch.
|
||||
onDebounced:
|
||||
body.options?.debounce && body.options?.resumeParentOnCompletion
|
||||
? async ({ existingRun, waitpoint, debounceKey }) => {
|
||||
return await this.traceEventConcern.traceDebouncedRun(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore,
|
||||
{
|
||||
existingRun,
|
||||
debounceKey,
|
||||
incomplete: waitpoint.status === "PENDING",
|
||||
isError: waitpoint.outputIsError,
|
||||
},
|
||||
async (spanEvent) => {
|
||||
const spanId =
|
||||
options?.parentAsLinkType === "replay"
|
||||
? spanEvent.spanId
|
||||
: spanEvent.traceparent?.spanId
|
||||
? `${spanEvent.traceparent.spanId}:${spanEvent.spanId}`
|
||||
: spanEvent.spanId;
|
||||
return spanId;
|
||||
}
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
this.prisma
|
||||
);
|
||||
|
||||
// If the returned run has a different friendlyId, it was debounced.
|
||||
// For triggerAndWait: stop the outer span since a replacement debounced span was created via onDebounced.
|
||||
// For regular trigger: let the span complete normally - no replacement span needed since the
|
||||
// original run already has its span from when it was first created.
|
||||
if (
|
||||
taskRun.friendlyId !== runFriendlyId &&
|
||||
body.options?.debounce &&
|
||||
body.options?.resumeParentOnCompletion
|
||||
) {
|
||||
event.stop();
|
||||
}
|
||||
|
||||
const error = taskRun.error ? TaskRunError.parse(taskRun.error) : undefined;
|
||||
|
||||
if (error) {
|
||||
event.failWithError(error);
|
||||
}
|
||||
|
||||
const result = { run: taskRun, error, isCached: false };
|
||||
|
||||
if (result?.error) {
|
||||
throw new ServiceValidationError(
|
||||
taskRunErrorToString(taskRunErrorEnhancer(result.error))
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof RunDuplicateIdempotencyKeyError) {
|
||||
//retry calling this function, because this time it will return the idempotent run
|
||||
return await this.call({
|
||||
taskId,
|
||||
environment,
|
||||
body,
|
||||
options: { ...options, runFriendlyId },
|
||||
attempt: attempt + 1,
|
||||
});
|
||||
}
|
||||
|
||||
if (error instanceof RunOneTimeUseTokenError) {
|
||||
throw new ServiceValidationError(
|
||||
`Cannot trigger ${taskId} with a one-time use token as it has already been used.`
|
||||
);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
);
|
||||
// Pipeline returned successfully — publish the claim if we held
|
||||
// one. Waiters polling for our key resolve to this runId.
|
||||
if (idempotencyClaim && result?.run?.friendlyId) {
|
||||
await publishMollifierClaim({
|
||||
envId: idempotencyClaim.envId,
|
||||
taskIdentifier: idempotencyClaim.taskIdentifier,
|
||||
idempotencyKey: idempotencyClaim.idempotencyKey,
|
||||
token: idempotencyClaim.token,
|
||||
runId: result.run.friendlyId,
|
||||
ttlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
// Pipeline threw — release the claim so the next claimant can
|
||||
// retry. Re-throw so the caller sees the original error.
|
||||
if (idempotencyClaim) {
|
||||
await releaseMollifierClaim(idempotencyClaim);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Build the engine.trigger() input object from the values gathered during
|
||||
// this.call(). Extracted so the mollify path can construct the
|
||||
// same input shape without re-entering the trace-run span. The pass-through
|
||||
// path spreads this result and attaches `onDebounced` inline; the mollify
|
||||
// path serialises it into the buffer for drainer replay.
|
||||
#buildEngineTriggerInput(args: {
|
||||
runFriendlyId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
body: TriggerTaskRequest["body"];
|
||||
options: TriggerTaskServiceOptions;
|
||||
queueName: string;
|
||||
lockedQueueId?: string;
|
||||
workerQueue?: string;
|
||||
region?: string;
|
||||
enableFastPath: boolean;
|
||||
lockedToBackgroundWorker?: {
|
||||
id: string;
|
||||
version: string;
|
||||
sdkVersion: string;
|
||||
cliVersion: string;
|
||||
};
|
||||
delayUntil?: Date;
|
||||
ttl?: string;
|
||||
metadataPacket?: { data?: string; dataType: string };
|
||||
tags: string[];
|
||||
depth: number;
|
||||
parentRun?: {
|
||||
id: string;
|
||||
rootTaskRunId?: string | null;
|
||||
queueTimestamp?: Date | null;
|
||||
taskEventStore?: string;
|
||||
};
|
||||
annotations: {
|
||||
triggerSource: string;
|
||||
triggerAction: string;
|
||||
rootTriggerSource: string;
|
||||
rootScheduleId?: string | undefined;
|
||||
};
|
||||
planType?: string;
|
||||
taskId: string;
|
||||
payloadPacket: { data?: string; dataType: string };
|
||||
traceContext: TriggerTraceContext;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
parentSpanId: string | undefined;
|
||||
taskEventStore: string;
|
||||
}) {
|
||||
return {
|
||||
friendlyId: args.runFriendlyId,
|
||||
environment: args.environment,
|
||||
idempotencyKey: args.idempotencyKey,
|
||||
idempotencyKeyExpiresAt: args.idempotencyKey ? args.idempotencyKeyExpiresAt : undefined,
|
||||
idempotencyKeyOptions: args.body.options?.idempotencyKeyOptions,
|
||||
taskIdentifier: args.taskId,
|
||||
payload: args.payloadPacket.data ?? "",
|
||||
payloadType: args.payloadPacket.dataType,
|
||||
context: args.body.context,
|
||||
traceContext: args.traceContext,
|
||||
traceId: args.traceId,
|
||||
spanId: args.spanId,
|
||||
parentSpanId: args.parentSpanId,
|
||||
replayedFromTaskRunFriendlyId: args.options.replayedFromTaskRunFriendlyId,
|
||||
lockedToVersionId: args.lockedToBackgroundWorker?.id,
|
||||
taskVersion: args.lockedToBackgroundWorker?.version,
|
||||
sdkVersion: args.lockedToBackgroundWorker?.sdkVersion,
|
||||
cliVersion: args.lockedToBackgroundWorker?.cliVersion,
|
||||
// Schema-level coercion now lands `body.options.concurrencyKey` as
|
||||
// `string` on the API path, but the BatchQueue worker rebuilds
|
||||
// body.options from Redis-stored items (Record<string, unknown>),
|
||||
// which can still carry the pre-fix shape from in-flight batches.
|
||||
concurrencyKey:
|
||||
typeof args.body.options?.concurrencyKey === "number"
|
||||
? String(args.body.options.concurrencyKey)
|
||||
: args.body.options?.concurrencyKey,
|
||||
queue: args.queueName,
|
||||
lockedQueueId: args.lockedQueueId,
|
||||
workerQueue: args.workerQueue,
|
||||
region: args.region,
|
||||
enableFastPath: args.enableFastPath,
|
||||
isTest: args.body.options?.test ?? false,
|
||||
delayUntil: args.delayUntil,
|
||||
queuedAt: args.delayUntil ? undefined : new Date(),
|
||||
maxAttempts: args.body.options?.maxAttempts,
|
||||
taskEventStore: args.taskEventStore,
|
||||
ttl: args.ttl,
|
||||
tags: args.tags,
|
||||
oneTimeUseToken: args.options.oneTimeUseToken,
|
||||
parentTaskRunId: args.parentRun?.id,
|
||||
rootTaskRunId: args.parentRun?.rootTaskRunId ?? args.parentRun?.id,
|
||||
batch: args.options?.batchId
|
||||
? { id: args.options.batchId, index: args.options.batchIndex ?? 0 }
|
||||
: undefined,
|
||||
resumeParentOnCompletion: args.body.options?.resumeParentOnCompletion,
|
||||
depth: args.depth,
|
||||
metadata: args.metadataPacket?.data,
|
||||
metadataType: args.metadataPacket?.dataType,
|
||||
seedMetadata: args.metadataPacket?.data,
|
||||
seedMetadataType: args.metadataPacket?.dataType,
|
||||
maxDurationInSeconds: args.body.options?.maxDuration
|
||||
? clampMaxDuration(args.body.options.maxDuration)
|
||||
: undefined,
|
||||
machine: args.body.options?.machine,
|
||||
priorityMs: args.body.options?.priority ? args.body.options.priority * 1_000 : undefined,
|
||||
queueTimestamp:
|
||||
args.options.queueTimestamp ??
|
||||
(args.parentRun && args.body.options?.resumeParentOnCompletion
|
||||
? (args.parentRun.queueTimestamp ?? undefined)
|
||||
: undefined),
|
||||
scheduleId: args.options.scheduleId,
|
||||
scheduleInstanceId: args.options.scheduleInstanceId,
|
||||
createdAt: args.options.overrideCreatedAt,
|
||||
bulkActionId: args.body.options?.bulkActionId,
|
||||
planType: args.planType,
|
||||
realtimeStreamsVersion: args.options.realtimeStreamsVersion,
|
||||
streamBasinName: args.environment.organization.streamBasinName,
|
||||
debounce: args.body.options?.debounce,
|
||||
annotations: args.annotations,
|
||||
};
|
||||
}
|
||||
|
||||
#propagateExternalTraceContext(
|
||||
traceContext: Record<string, unknown>,
|
||||
parentRunTraceContext: unknown,
|
||||
parentSpanId: string | undefined
|
||||
): TriggerTraceContext {
|
||||
if (!parentRunTraceContext) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const parsedParentRunTraceContext = TriggerTraceContext.safeParse(parentRunTraceContext);
|
||||
|
||||
if (!parsedParentRunTraceContext.success) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const { external } = parsedParentRunTraceContext.data;
|
||||
|
||||
if (!external) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
if (!external.traceparent) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const parsedTraceparent = parseTraceparent(external.traceparent);
|
||||
|
||||
if (!parsedTraceparent) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const newExternalTraceparent = serializeTraceparent(
|
||||
parsedTraceparent.traceId,
|
||||
parentSpanId ?? parsedTraceparent.spanId,
|
||||
parsedTraceparent.traceFlags
|
||||
);
|
||||
|
||||
return {
|
||||
...traceContext,
|
||||
external: {
|
||||
...external,
|
||||
traceparent: newExternalTraceparent,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
import type { BackgroundWorker, TaskRun } from "@trigger.dev/database";
|
||||
import type { IOPacket, TaskRunError, TriggerTaskRequestBody } from "@trigger.dev/core/v3";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import type { ReportUsagePlan } from "@trigger.dev/platform";
|
||||
|
||||
export type TriggerTaskServiceOptions = {
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, unknown>;
|
||||
spanParentAsLink?: boolean;
|
||||
parentAsLinkType?: "replay" | "trigger";
|
||||
batchId?: string;
|
||||
batchIndex?: number;
|
||||
customIcon?: string;
|
||||
runFriendlyId?: string;
|
||||
skipChecks?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
overrideCreatedAt?: Date;
|
||||
planType?: string;
|
||||
};
|
||||
|
||||
// domain/triggerTask.ts
|
||||
export type TriggerTaskRequest = {
|
||||
taskId: string;
|
||||
friendlyId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
body: TriggerTaskRequestBody;
|
||||
options?: TriggerTaskServiceOptions;
|
||||
};
|
||||
|
||||
export type TriggerTaskResult = {
|
||||
run: TaskRun;
|
||||
isCached: boolean;
|
||||
error?: TaskRunError;
|
||||
};
|
||||
|
||||
export type QueueValidationResult =
|
||||
| {
|
||||
ok: true;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
maximumSize: number;
|
||||
queueSize: number;
|
||||
};
|
||||
|
||||
export type QueueProperties = {
|
||||
queueName: string;
|
||||
lockedQueueId?: string;
|
||||
taskTtl?: string | null;
|
||||
taskKind?: string;
|
||||
};
|
||||
|
||||
export type LockedBackgroundWorker = Pick<
|
||||
BackgroundWorker,
|
||||
"id" | "version" | "sdkVersion" | "cliVersion"
|
||||
>;
|
||||
|
||||
// Core domain interfaces
|
||||
export interface QueueManager {
|
||||
resolveQueueProperties(
|
||||
request: TriggerTaskRequest,
|
||||
lockedBackgroundWorker?: LockedBackgroundWorker
|
||||
): Promise<QueueProperties>;
|
||||
validateQueueLimits(
|
||||
env: AuthenticatedEnvironment,
|
||||
queueName: string,
|
||||
itemsToAdd?: number
|
||||
): Promise<QueueValidationResult>;
|
||||
getWorkerQueue(
|
||||
env: AuthenticatedEnvironment,
|
||||
regionOverride?: string
|
||||
): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined>;
|
||||
}
|
||||
|
||||
export interface PayloadProcessor {
|
||||
process(request: TriggerTaskRequest): Promise<IOPacket>;
|
||||
}
|
||||
|
||||
export interface TagValidationParams {
|
||||
tags?: string[] | string;
|
||||
}
|
||||
|
||||
export interface EntitlementValidationParams {
|
||||
environment: AuthenticatedEnvironment;
|
||||
}
|
||||
|
||||
export interface MaxAttemptsValidationParams {
|
||||
taskId: string;
|
||||
attempt: number;
|
||||
}
|
||||
|
||||
export interface ParentRunValidationParams {
|
||||
taskId: string;
|
||||
parentRun?: TaskRun;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
}
|
||||
|
||||
export type ValidationResult =
|
||||
| {
|
||||
ok: true;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: Error;
|
||||
};
|
||||
|
||||
export type EntitlementValidationResult =
|
||||
| {
|
||||
ok: true;
|
||||
plan?: ReportUsagePlan;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: Error;
|
||||
};
|
||||
|
||||
export interface TriggerTaskValidator {
|
||||
validateTags(params: TagValidationParams): ValidationResult;
|
||||
validateEntitlement(params: EntitlementValidationParams): Promise<EntitlementValidationResult>;
|
||||
validateMaxAttempts(params: MaxAttemptsValidationParams): ValidationResult;
|
||||
validateParentRun(params: ParentRunValidationParams): ValidationResult;
|
||||
}
|
||||
|
||||
export type TracedEventSpan = {
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
traceContext: Record<string, unknown>;
|
||||
traceparent?: {
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
};
|
||||
setAttribute: (key: string, value: string) => void;
|
||||
failWithError: (error: TaskRunError) => void;
|
||||
/**
|
||||
* Stop the span without writing any event.
|
||||
* Used when a debounced run is returned - the span for the debounced
|
||||
* trigger is created separately via traceDebouncedRun.
|
||||
*/
|
||||
stop: () => void;
|
||||
};
|
||||
|
||||
export interface TraceEventConcern {
|
||||
traceRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T>;
|
||||
traceIdempotentRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
idempotencyKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T>;
|
||||
traceDebouncedRun<T>(
|
||||
request: TriggerTaskRequest,
|
||||
parentStore: string | undefined,
|
||||
options: {
|
||||
existingRun: TaskRun;
|
||||
debounceKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T>;
|
||||
}
|
||||
|
||||
export type TriggerRacepoints = "idempotencyKey";
|
||||
|
||||
export interface TriggerRacepointSystem {
|
||||
waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getEntitlement } from "~/services/platform.v3.server";
|
||||
import { MAX_ATTEMPTS } from "~/v3/services/triggerTask.server";
|
||||
import { isFinalRunStatus } from "~/v3/taskStatus";
|
||||
import type {
|
||||
EntitlementValidationParams,
|
||||
EntitlementValidationResult,
|
||||
MaxAttemptsValidationParams,
|
||||
ParentRunValidationParams,
|
||||
TagValidationParams,
|
||||
TriggerTaskValidator,
|
||||
ValidationResult,
|
||||
} from "../types";
|
||||
import { validateProductionEntitlement } from "./validateProductionEntitlement.server";
|
||||
import { ServiceValidationError } from "~/v3/services/common.server";
|
||||
|
||||
export class DefaultTriggerTaskValidator implements TriggerTaskValidator {
|
||||
validateTags(params: TagValidationParams): ValidationResult {
|
||||
const { tags } = params;
|
||||
|
||||
if (!tags) {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (typeof tags === "string") {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (tags.length > MAX_TAGS_PER_RUN) {
|
||||
return {
|
||||
ok: false,
|
||||
error: new ServiceValidationError(
|
||||
`Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${tags.length}.`
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async validateEntitlement(
|
||||
params: EntitlementValidationParams
|
||||
): Promise<EntitlementValidationResult> {
|
||||
return validateProductionEntitlement(params, getEntitlement);
|
||||
}
|
||||
|
||||
validateMaxAttempts(params: MaxAttemptsValidationParams): ValidationResult {
|
||||
const { taskId, attempt } = params;
|
||||
|
||||
if (attempt > MAX_ATTEMPTS) {
|
||||
return {
|
||||
ok: false,
|
||||
error: new ServiceValidationError(
|
||||
`Failed to trigger ${taskId} after ${MAX_ATTEMPTS} attempts.`
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
validateParentRun(params: ParentRunValidationParams): ValidationResult {
|
||||
const { taskId, parentRun, resumeParentOnCompletion } = params;
|
||||
|
||||
// If there's no parent run specified, that's fine
|
||||
if (!parentRun) {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// If we're not resuming the parent, we don't need to validate its status
|
||||
if (!resumeParentOnCompletion) {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// Check if the parent run is in a final state
|
||||
if (isFinalRunStatus(parentRun.status)) {
|
||||
logger.debug("Parent run is in a terminal state", {
|
||||
parentRun,
|
||||
});
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
error: new ServiceValidationError(
|
||||
`Cannot trigger ${taskId} as the parent run has a status of ${parentRun.status}`
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { EntitlementResult } from "~/services/billingLimit.schemas";
|
||||
import { OutOfEntitlementError } from "~/v3/outOfEntitlementError.server";
|
||||
import type { EntitlementValidationParams, EntitlementValidationResult } from "../types";
|
||||
|
||||
export type GetEntitlementFn = (organizationId: string) => Promise<EntitlementResult | undefined>;
|
||||
|
||||
export async function validateProductionEntitlement(
|
||||
params: EntitlementValidationParams,
|
||||
getEntitlementFn: GetEntitlementFn
|
||||
): Promise<EntitlementValidationResult> {
|
||||
const { environment } = params;
|
||||
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
const result = await getEntitlementFn(environment.organizationId);
|
||||
|
||||
if (result && result.hasAccess === false) {
|
||||
return {
|
||||
ok: false,
|
||||
error: new OutOfEntitlementError(),
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, plan: result?.plan };
|
||||
}
|
||||
Reference in New Issue
Block a user