chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
+110
View File
@@ -0,0 +1,110 @@
# @internal/schedule-engine
The `@internal/schedule-engine` package encapsulates all scheduling logic for Trigger.dev, providing a clean API boundary for managing scheduled tasks and their execution.
## Architecture
The ScheduleEngine follows the same pattern as the RunEngine, providing:
- **Centralized Schedule Management**: All schedule-related operations go through the ScheduleEngine
- **Redis Worker Integration**: Built-in Redis-based distributed task scheduling
- **Distributed Execution**: Prevents thundering herd issues by distributing executions across time windows
- **Comprehensive Testing**: Built-in utilities for testing schedule behavior
## Key Components
### ScheduleEngine Class
The main interface for all schedule operations:
```typescript
import { ScheduleEngine } from "@internal/schedule-engine";
const engine = new ScheduleEngine({
prisma,
redis: {
/* Redis configuration */
},
worker: {
/* Worker configuration */
},
distributionWindow: { seconds: 30 }, // Optional: default 30s
});
// Register next schedule instance
await engine.registerNextTaskScheduleInstance({ instanceId });
// Upsert a schedule
await engine.upsertTaskSchedule({
projectId,
schedule: {
taskIdentifier: "my-task",
cron: "0 */5 * * *",
timezone: "UTC",
environments: ["env-1", "env-2"],
},
});
```
### Distributed Scheduling
The engine includes built-in distributed scheduling to prevent all scheduled tasks from executing at exactly the same moment:
```typescript
import { calculateDistributedExecutionTime } from "@internal/schedule-engine";
const exactTime = new Date("2024-01-01T12:00:00Z");
const distributedTime = calculateDistributedExecutionTime(exactTime, 30); // 30-second window
```
### Schedule Calculation
High-performance CRON schedule calculation with optimization for old timestamps:
```typescript
import {
calculateNextScheduledTimestampFromNow,
nextScheduledTimestamps,
} from "@internal/schedule-engine";
const nextRun = calculateNextScheduledTimestampFromNow("0 */5 * * *", "UTC");
const upcoming = nextScheduledTimestamps("0 */5 * * *", "UTC", nextRun, 5);
```
## Integration with Webapp
The ScheduleEngine should be the **API boundary** between the webapp and schedule logic. Services in the webapp should call into the ScheduleEngine rather than implementing schedule logic directly.
### Migration Path
Currently, the webapp uses individual services like:
- `RegisterNextTaskScheduleInstanceService`
- `TriggerScheduledTaskService`
- Schedule calculation utilities
These should be replaced with ScheduleEngine method calls:
```typescript
// Old approach
const service = new RegisterNextTaskScheduleInstanceService(tx);
await service.call(instanceId);
// New approach
await scheduleEngine.registerNextTaskScheduleInstance({ instanceId });
```
## Configuration
The ScheduleEngine expects these configuration options:
- `prisma`: PrismaClient instance
- `redis`: Redis connection configuration
- `worker`: Worker configuration (concurrency, polling intervals)
- `distributionWindow`: Optional time window for distributed execution
- `tracer`: Optional OpenTelemetry tracer
- `meter`: Optional OpenTelemetry meter
## Testing
The package includes comprehensive test utilities and examples. See the test directory for usage examples.
@@ -0,0 +1,39 @@
{
"name": "@internal/schedule-engine",
"private": true,
"version": "0.0.1",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"type": "module",
"exports": {
".": {
"@triggerdotdev/source": "./src/index.ts",
"import": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"default": "./dist/src/index.js"
}
},
"dependencies": {
"@internal/redis": "workspace:*",
"@trigger.dev/redis-worker": "workspace:*",
"@internal/tracing": "workspace:*",
"@trigger.dev/core": "workspace:*",
"@trigger.dev/database": "workspace:*",
"cron-parser": "^4.9.0",
"cronstrue": "^2.50.0",
"nanoid": "3.3.8",
"zod": "3.25.76"
},
"devDependencies": {
"@internal/testcontainers": "workspace:*",
"rimraf": "6.0.1"
},
"scripts": {
"clean": "rimraf dist",
"typecheck": "tsc --noEmit -p tsconfig.build.json",
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled",
"build": "pnpm run clean && tsc -p tsconfig.build.json",
"dev": "tsc --watch -p tsconfig.build.json"
}
}
@@ -0,0 +1,35 @@
/**
* Calculates a distributed execution time for a scheduled task.
* Tasks are distributed across a time window before the exact schedule time
* to prevent thundering herd issues while maintaining schedule accuracy.
*/
export function calculateDistributedExecutionTime(
exactScheduleTime: Date,
distributionWindowSeconds: number = 30,
instanceId?: string
): Date {
// Create seed by combining ISO timestamp with optional instanceId
// This ensures different instances get different distributions even with same schedule time
const timeSeed = exactScheduleTime.toISOString();
const seed = instanceId ? `${timeSeed}:${instanceId}` : timeSeed;
// Use a better hash function (FNV-1a variant) for more uniform distribution
let hash = 2166136261; // FNV offset basis (32-bit)
for (let i = 0; i < seed.length; i++) {
hash ^= seed.charCodeAt(i);
hash *= 16777619; // FNV prime (32-bit)
// Keep it as 32-bit unsigned integer
hash = hash >>> 0;
}
// Convert hash to a value between 0 and 1 using better normalization
// Use the full 32-bit range for better distribution
const normalized = hash / 0xffffffff;
// Calculate offset in milliseconds (0 to distributionWindowSeconds * 1000)
const offsetMs = Math.floor(normalized * distributionWindowSeconds * 1000);
// Return time that's offsetMs before the exact schedule time
return new Date(exactScheduleTime.getTime() - offsetMs);
}
@@ -0,0 +1,867 @@
import type { Counter, Histogram, Meter, Tracer } from "@internal/tracing";
import { getMeter, getTracer, startSpan } from "@internal/tracing";
import { Logger } from "@trigger.dev/core/logger";
import type { PrismaClient } from "@trigger.dev/database";
import { Worker, type JobHandlerParams } from "@trigger.dev/redis-worker";
import { calculateDistributedExecutionTime } from "./distributedScheduling.js";
import {
calculateNextScheduledTimestamp,
nextScheduledTimestamps,
previousScheduledTimestamp,
} from "./scheduleCalculation.js";
import type {
RegisterScheduleInstanceParams,
ScheduleEngineOptions,
TriggerScheduledTaskCallback,
TriggerScheduleParams,
} from "./types.js";
import { scheduleWorkerCatalog } from "./workerCatalog.js";
import { tryCatch } from "@trigger.dev/core/utils";
export class ScheduleEngine {
private worker: Worker<typeof scheduleWorkerCatalog>;
private logger: Logger;
private tracer: Tracer;
private meter: Meter;
private distributionWindowSeconds: number;
// Metrics
private scheduleRegistrationCounter: Counter;
private scheduleExecutionCounter: Counter;
private scheduleExecutionDuration: Histogram;
private scheduleExecutionFailureCounter: Counter;
private distributionOffsetHistogram: Histogram;
private devEnvironmentCheckCounter: Counter;
prisma: PrismaClient;
private onTriggerScheduledTask: TriggerScheduledTaskCallback;
constructor(private readonly options: ScheduleEngineOptions) {
this.logger =
options.logger ?? new Logger("ScheduleEngine", (this.options.logLevel ?? "info") as any);
this.prisma = options.prisma;
this.distributionWindowSeconds = options.distributionWindow?.seconds ?? 30;
this.onTriggerScheduledTask = options.onTriggerScheduledTask;
this.tracer = options.tracer ?? getTracer("schedule-engine");
this.meter = options.meter ?? getMeter("schedule-engine");
// Initialize metrics
this.scheduleRegistrationCounter = this.meter.createCounter("schedule_registrations_total", {
description: "Total number of schedule registrations",
});
this.scheduleExecutionCounter = this.meter.createCounter("schedule_executions_total", {
description: "Total number of schedule executions",
});
this.scheduleExecutionDuration = this.meter.createHistogram("schedule_execution_duration_ms", {
description: "Duration of schedule execution in milliseconds",
unit: "ms",
});
this.scheduleExecutionFailureCounter = this.meter.createCounter(
"schedule_execution_failures_total",
{
description: "Total number of schedule execution failures",
}
);
this.distributionOffsetHistogram = this.meter.createHistogram(
"schedule_distribution_offset_ms",
{
description: "Distribution offset from exact schedule time in milliseconds",
unit: "ms",
}
);
this.devEnvironmentCheckCounter = this.meter.createCounter("dev_environment_checks_total", {
description: "Total number of development environment connectivity checks",
});
this.worker = new Worker({
name: "schedule-engine-worker",
redisOptions: {
...options.redis,
keyPrefix: `${options.redis.keyPrefix ?? ""}schedule:`,
},
catalog: scheduleWorkerCatalog,
concurrency: {
limit: options.worker.concurrency,
workers: options.worker.workers,
tasksPerWorker: options.worker.tasksPerWorker,
},
pollIntervalMs: options.worker.pollIntervalMs,
shutdownTimeoutMs: options.worker.shutdownTimeoutMs,
logger: new Logger("ScheduleEngineWorker", (options.logLevel ?? "info") as any),
jobs: {
"schedule.triggerScheduledTask": this.#handleTriggerScheduledTaskJob.bind(this),
},
});
if (!options.worker.disabled) {
this.worker.start();
this.logger.info("Schedule engine worker started", {
concurrency: options.worker.concurrency,
pollIntervalMs: options.worker.pollIntervalMs,
distributionWindowSeconds: this.distributionWindowSeconds,
});
} else {
this.logger.info("Schedule engine worker disabled");
}
}
/**
* Registers the next scheduled instance for a schedule
*/
async registerNextTaskScheduleInstance(params: RegisterScheduleInstanceParams) {
return startSpan(this.tracer, "registerNextTaskScheduleInstance", async (span) => {
const startTime = Date.now();
if (this.options.onRegisterScheduleInstance) {
const [registerError] = await tryCatch(
this.options.onRegisterScheduleInstance(params.instanceId)
);
if (registerError) {
this.logger.error("Error calling the onRegisterScheduleInstance callback", {
instanceId: params.instanceId,
error: registerError,
});
}
}
span.setAttribute("instanceId", params.instanceId);
this.logger.debug("Starting schedule registration", {
instanceId: params.instanceId,
});
try {
const instance = await this.prisma.taskScheduleInstance.findFirst({
where: {
id: params.instanceId,
},
include: {
taskSchedule: true,
environment: true,
},
});
if (!instance) {
this.logger.warn("Schedule instance not found during registration", {
instanceId: params.instanceId,
});
span.setAttribute("error", "instance_not_found");
return;
}
span.setAttribute("task_schedule_id", instance.taskSchedule.id);
span.setAttribute("task_schedule_instance_id", instance.id);
span.setAttribute("task_identifier", instance.taskSchedule.taskIdentifier);
span.setAttribute("environment_type", instance.environment.type);
span.setAttribute("schedule_active", instance.active);
span.setAttribute("task_schedule_active", instance.taskSchedule.active);
span.setAttribute(
"task_schedule_generator_expression",
instance.taskSchedule.generatorExpression
);
const fromTimestamp = params.fromTimestamp ?? new Date();
span.setAttribute("from_timestamp", fromTimestamp.toISOString());
const nextScheduledTimestamp = calculateNextScheduledTimestamp(
instance.taskSchedule.generatorExpression,
instance.taskSchedule.timezone,
fromTimestamp
);
span.setAttribute("next_scheduled_timestamp", nextScheduledTimestamp.toISOString());
const schedulingDelayMs = nextScheduledTimestamp.getTime() - Date.now();
span.setAttribute("scheduling_delay_ms", schedulingDelayMs);
this.logger.debug("Calculated next schedule timestamp", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
nextScheduledTimestamp: nextScheduledTimestamp.toISOString(),
schedulingDelayMs,
generatorExpression: instance.taskSchedule.generatorExpression,
timezone: instance.taskSchedule.timezone,
});
// Determine the lastScheduleTime to embed in the next worker job's
// payload. If the caller passed it explicitly (the after-fire path
// does this with the just-fired timestamp, the after-skip path
// carries the existing value forward), use that. Otherwise — every
// external caller (deploy sync, schedule upsert, recovery) — derive
// from the cron expression's previous slot.
//
// Without this fallback, every deploy / cron edit would clobber the
// existing in-flight job's lastScheduleTime with `undefined`, and
// the next fire would surface a frozen DB-column value to the
// customer (since this PR stops writing that column). Pure cron
// math, no DB read on top of the existing instance load — the
// recovery loop already pays the cost of loading the instance.
let lastScheduleTime = params.lastScheduleTime;
if (lastScheduleTime === undefined) {
try {
const cronPrev = previousScheduledTimestamp(
instance.taskSchedule.generatorExpression,
instance.taskSchedule.timezone
);
// Guarded against the cron's previous slot predating the
// instance itself — for a brand-new schedule, the slot is from
// before the schedule existed, so `undefined` is the honest
// answer (preserves the `if (!payload.lastTimestamp)` first-run
// sentinel customers rely on).
if (cronPrev.getTime() > instance.createdAt.getTime()) {
lastScheduleTime = cronPrev;
}
} catch {
// Malformed cron — leave undefined.
}
}
await this.enqueueScheduledTask(
params.instanceId,
nextScheduledTimestamp,
lastScheduleTime
);
// Record metrics
this.scheduleRegistrationCounter.add(1, {
environment_type: instance.environment.type,
schedule_type: instance.taskSchedule.type,
});
const duration = Date.now() - startTime;
this.logger.debug("Schedule registration completed", {
instanceId: params.instanceId,
durationMs: duration,
});
span.setAttribute("success", true);
span.setAttribute("duration_ms", duration);
} catch (error) {
const duration = Date.now() - startTime;
this.logger.error("Failed to register schedule instance", {
instanceId: params.instanceId,
durationMs: duration,
error: error instanceof Error ? error.message : String(error),
});
span.setAttribute("error", true);
span.setAttribute("error_message", error instanceof Error ? error.message : String(error));
span.setAttribute("duration_ms", duration);
throw error;
}
});
}
async #handleTriggerScheduledTaskJob({
payload,
}: JobHandlerParams<typeof scheduleWorkerCatalog, "schedule.triggerScheduledTask">) {
await this.triggerScheduledTask({
instanceId: payload.instanceId,
finalAttempt: false, // TODO: implement retry logic
exactScheduleTime: payload.exactScheduleTime,
lastScheduleTime: payload.lastScheduleTime,
});
}
/**
* Triggers a scheduled task (called by the Redis worker)
*/
async triggerScheduledTask(params: TriggerScheduleParams) {
return startSpan(this.tracer, "triggerScheduledTask", async (span) => {
const startTime = Date.now();
span.setAttribute("instanceId", params.instanceId);
span.setAttribute("finalAttempt", params.finalAttempt);
if (params.exactScheduleTime) {
span.setAttribute("exactScheduleTime", params.exactScheduleTime.toISOString());
}
this.logger.debug("Starting scheduled task trigger", {
instanceId: params.instanceId,
finalAttempt: params.finalAttempt,
exactScheduleTime: params.exactScheduleTime?.toISOString(),
});
let taskIdentifier: string | undefined;
let environmentType: string | undefined;
let scheduleType: string | undefined;
try {
const instance = await this.prisma.taskScheduleInstance.findFirst({
where: {
id: params.instanceId,
},
include: {
taskSchedule: true,
environment: {
include: {
project: true,
organization: true,
orgMember: true,
},
},
},
});
if (!instance) {
this.logger.debug("Schedule instance not found", {
instanceId: params.instanceId,
});
span.setAttribute("error", "instance_not_found");
return;
}
taskIdentifier = instance.taskSchedule.taskIdentifier;
environmentType = instance.environment.type;
scheduleType = instance.taskSchedule.type;
span.setAttribute("task_identifier", taskIdentifier);
span.setAttribute("environment_type", environmentType);
span.setAttribute("schedule_type", scheduleType);
span.setAttribute("organization_id", instance.environment.organization.id);
span.setAttribute("project_id", instance.environment.project.id);
span.setAttribute("environment_id", instance.environment.id);
// Check if organization/project/environment is still valid
if (instance.environment.organization.deletedAt) {
this.logger.debug("Organization is deleted, skipping schedule", {
instanceId: params.instanceId,
scheduleId: instance.taskSchedule.friendlyId,
organizationId: instance.environment.organization.id,
});
span.setAttribute("skip_reason", "organization_deleted");
return;
}
if (instance.environment.project.deletedAt) {
this.logger.debug("Project is deleted, skipping schedule", {
instanceId: params.instanceId,
scheduleId: instance.taskSchedule.friendlyId,
projectId: instance.environment.project.id,
});
span.setAttribute("skip_reason", "project_deleted");
return;
}
if (instance.environment.archivedAt) {
this.logger.debug("Environment is archived, skipping schedule", {
instanceId: params.instanceId,
scheduleId: instance.taskSchedule.friendlyId,
environmentId: instance.environment.id,
});
span.setAttribute("skip_reason", "environment_archived");
return;
}
let shouldTrigger = true;
let skipReason: string | undefined;
if (!instance.active || !instance.taskSchedule.active) {
this.logger.debug("Schedule is inactive", {
instanceId: params.instanceId,
instanceActive: instance.active,
scheduleActive: instance.taskSchedule.active,
});
shouldTrigger = false;
skipReason = "schedule_inactive";
}
// For development environments, check if there's an active session
if (instance.environment.type === "DEVELOPMENT") {
this.devEnvironmentCheckCounter.add(1, {
environment_id: instance.environment.id,
});
const [devConnectedError, isConnected] = await tryCatch(
this.options.isDevEnvironmentConnectedHandler(instance.environment.id)
);
if (devConnectedError) {
this.logger.error("Error checking if development environment is connected", {
instanceId: params.instanceId,
environmentId: instance.environment.id,
error: devConnectedError,
});
span.setAttribute("dev_connection_check_error", true);
shouldTrigger = false;
skipReason = "dev_connection_check_failed";
} else if (!isConnected) {
this.logger.debug("Development environment is disconnected", {
instanceId: params.instanceId,
environmentId: instance.environment.id,
});
span.setAttribute("dev_connected", false);
shouldTrigger = false;
skipReason = "dev_disconnected";
} else {
span.setAttribute("dev_connected", true);
}
}
span.setAttribute("should_trigger", shouldTrigger);
if (skipReason) {
span.setAttribute("skip_reason", skipReason);
}
// Calculate the schedule timestamp that will be used (regardless of whether we trigger or not)
const scheduleTimestamp = params.exactScheduleTime ?? new Date();
if (shouldTrigger) {
// payload.lastTimestamp is the actual previous fire time. Sources, in
// order:
// 1. params.lastScheduleTime — populated by the engine when this
// job was enqueued. Always present for jobs enqueued post-deploy.
// 2. instance.lastScheduledTimestamp — backward-compat fallback for
// in-flight Redis jobs enqueued by older engines that didn't
// include lastScheduleTime in the payload. Once those drain
// this fallback never triggers and we can drop the column.
// 3. undefined — first-ever fire (no previous fire to point at).
const lastTimestamp =
params.lastScheduleTime ?? instance.lastScheduledTimestamp ?? undefined;
const payload = {
scheduleId: instance.taskSchedule.friendlyId,
type: instance.taskSchedule.type as "DECLARATIVE" | "IMPERATIVE",
timestamp: scheduleTimestamp,
lastTimestamp,
externalId: instance.taskSchedule.externalId ?? undefined,
timezone: instance.taskSchedule.timezone,
upcoming: nextScheduledTimestamps(
instance.taskSchedule.generatorExpression,
instance.taskSchedule.timezone,
scheduleTimestamp,
10
),
};
// Calculate execution timing metrics
const actualExecutionTime = new Date();
const schedulingAccuracyMs = actualExecutionTime.getTime() - scheduleTimestamp.getTime();
span.setAttribute("scheduling_accuracy_ms", schedulingAccuracyMs);
span.setAttribute("actual_execution_time", actualExecutionTime.toISOString());
this.logger.debug("Triggering scheduled task", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
scheduleTimestamp: scheduleTimestamp.toISOString(),
actualExecutionTime: actualExecutionTime.toISOString(),
schedulingAccuracyMs,
lastTimestamp: lastTimestamp?.toISOString(),
});
const triggerStartTime = Date.now();
// Rewritten try/catch to use tryCatch utility
const [triggerError, result] = await tryCatch(
this.onTriggerScheduledTask({
taskIdentifier: instance.taskSchedule.taskIdentifier,
environment: instance.environment,
payload,
scheduleInstanceId: instance.id,
scheduleId: instance.taskSchedule.id,
exactScheduleTime: scheduleTimestamp,
})
);
const triggerDuration = Date.now() - triggerStartTime;
this.scheduleExecutionDuration.record(triggerDuration, {
environment_type: environmentType,
schedule_type: scheduleType,
});
if (triggerError) {
this.logger.error("Error calling trigger callback", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
durationMs: triggerDuration,
error: triggerError instanceof Error ? triggerError.message : String(triggerError),
});
this.scheduleExecutionFailureCounter.add(1, {
environment_type: environmentType,
schedule_type: scheduleType,
error_type: "callback_error",
});
span.setAttribute("trigger_error", true);
span.setAttribute(
"trigger_error_message",
triggerError instanceof Error ? triggerError.message : String(triggerError)
);
} else if (result) {
if (result.success) {
this.logger.debug("Successfully triggered scheduled task", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
durationMs: triggerDuration,
});
this.scheduleExecutionCounter.add(1, {
environment_type: environmentType,
schedule_type: scheduleType,
status: "success",
});
span.setAttribute("trigger_success", true);
} else {
// QUEUE_LIMIT and OUT_OF_ENTITLEMENTS are expected,
// non-actionable outcomes (the environment is at its queue limit,
// or the org is out of entitlements). Log them as warnings so they
// aren't reported as errors, while still recording the metric.
const isExpectedFailure =
result.errorType === "QUEUE_LIMIT" || result.errorType === "OUT_OF_ENTITLEMENTS";
if (isExpectedFailure) {
this.logger.warn("Scheduled task trigger skipped", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
durationMs: triggerDuration,
errorType: result.errorType,
error: result.error,
});
} else {
this.logger.error("Failed to trigger scheduled task", {
instanceId: params.instanceId,
taskIdentifier: instance.taskSchedule.taskIdentifier,
durationMs: triggerDuration,
error: result.error,
});
}
const failureErrorType =
result.errorType === "QUEUE_LIMIT"
? "queue_limit"
: result.errorType === "OUT_OF_ENTITLEMENTS"
? "out_of_entitlements"
: "task_failure";
this.scheduleExecutionFailureCounter.add(1, {
environment_type: environmentType,
schedule_type: scheduleType,
error_type: failureErrorType,
});
span.setAttribute("trigger_success", false);
if (result.error) {
span.setAttribute("trigger_error_message", result.error);
}
}
}
span.setAttribute("trigger_duration_ms", triggerDuration);
} else {
this.logger.debug("Skipping task trigger due to conditions", {
instanceId: params.instanceId,
reason: skipReason,
});
this.scheduleExecutionCounter.add(1, {
environment_type: environmentType ?? "unknown",
schedule_type: scheduleType ?? "unknown",
status: "skipped",
});
}
// Register the next run. `fromTimestamp` advances on every tick so
// the next cron slot keeps marching forward even through skips.
// `lastScheduleTime` is the actual previous fire time the next job
// will report as `payload.lastTimestamp` — only advance it when we
// actually triggered, otherwise carry forward the existing value so
// a long pause/disconnect doesn't quietly overwrite the real
// last-fire timestamp with a series of skipped slots.
const carriedLastScheduleTime = shouldTrigger
? scheduleTimestamp
: (params.lastScheduleTime ?? instance.lastScheduledTimestamp ?? undefined);
const [nextRunError] = await tryCatch(
this.registerNextTaskScheduleInstance({
instanceId: params.instanceId,
fromTimestamp: scheduleTimestamp,
lastScheduleTime: carriedLastScheduleTime,
})
);
if (nextRunError) {
this.logger.error("Failed to schedule next run after execution", {
instanceId: params.instanceId,
error: nextRunError instanceof Error ? nextRunError.message : String(nextRunError),
});
span.setAttribute("next_run_registration_error", true);
span.setAttribute(
"next_run_error_message",
nextRunError instanceof Error ? nextRunError.message : String(nextRunError)
);
if (!params.finalAttempt) {
throw nextRunError;
}
} else {
span.setAttribute("next_run_registered", true);
}
const totalDuration = Date.now() - startTime;
this.logger.debug("Scheduled task trigger completed", {
instanceId: params.instanceId,
totalDurationMs: totalDuration,
});
span.setAttribute("total_duration_ms", totalDuration);
span.setAttribute("success", true);
} catch (error) {
const totalDuration = Date.now() - startTime;
this.logger.error("Failed to trigger scheduled task", {
instanceId: params.instanceId,
totalDurationMs: totalDuration,
error: error instanceof Error ? error.message : String(error),
});
this.scheduleExecutionFailureCounter.add(1, {
environment_type: environmentType ?? "unknown",
schedule_type: scheduleType ?? "unknown",
error_type: "system_error",
});
span.setAttribute("error", true);
span.setAttribute("error_message", error instanceof Error ? error.message : String(error));
span.setAttribute("total_duration_ms", totalDuration);
throw error;
}
});
}
/**
* Enqueues a scheduled task with distributed execution timing
*/
private async enqueueScheduledTask(
instanceId: string,
exactScheduleTime: Date,
lastScheduleTime?: Date
) {
return startSpan(this.tracer, "enqueueScheduledTask", async (span) => {
span.setAttribute("instanceId", instanceId);
span.setAttribute("exactScheduleTime", exactScheduleTime.toISOString());
if (lastScheduleTime) {
span.setAttribute("lastScheduleTime", lastScheduleTime.toISOString());
}
const distributedExecutionTime = calculateDistributedExecutionTime(
exactScheduleTime,
this.distributionWindowSeconds,
instanceId
);
const distributionOffsetMs = exactScheduleTime.getTime() - distributedExecutionTime.getTime();
span.setAttribute("distributedExecutionTime", distributedExecutionTime.toISOString());
span.setAttribute("distributionOffsetMs", distributionOffsetMs);
span.setAttribute("distributionWindowSeconds", this.distributionWindowSeconds);
this.distributionOffsetHistogram.record(distributionOffsetMs, {
distribution_window_seconds: this.distributionWindowSeconds.toString(),
});
this.logger.debug("Enqueuing scheduled task with distributed execution", {
instanceId,
exactScheduleTime: exactScheduleTime.toISOString(),
distributedExecutionTime: distributedExecutionTime.toISOString(),
distributionOffsetMs,
distributionWindowSeconds: this.distributionWindowSeconds,
});
try {
await this.worker.enqueue({
id: `scheduled-task-instance:${instanceId}`,
job: "schedule.triggerScheduledTask",
payload: {
instanceId,
exactScheduleTime,
lastScheduleTime,
},
availableAt: distributedExecutionTime,
});
span.setAttribute("enqueue_success", true);
this.logger.debug("Successfully enqueued scheduled task", {
instanceId,
jobId: `scheduled-task-instance:${instanceId}`,
});
} catch (error) {
this.logger.error("Failed to enqueue scheduled task", {
instanceId,
error: error instanceof Error ? error.message : String(error),
});
span.setAttribute("enqueue_error", true);
span.setAttribute(
"enqueue_error_message",
error instanceof Error ? error.message : String(error)
);
throw error;
}
});
}
public recoverSchedulesInEnvironment(projectId: string, environmentId: string) {
return startSpan(this.tracer, "recoverSchedulesInEnvironment", async (span) => {
this.logger.info("Recovering schedules in environment", {
environmentId,
projectId,
});
span.setAttribute("environmentId", environmentId);
const schedules = await this.prisma.taskSchedule.findMany({
where: {
projectId,
instances: {
some: {
environmentId,
},
},
},
select: {
id: true,
generatorExpression: true,
timezone: true,
instances: {
select: {
id: true,
environmentId: true,
createdAt: true,
},
},
},
});
const instancesWithSchedule = schedules
.map((schedule) => ({
schedule,
instance: schedule.instances.find((instance) => instance.environmentId === environmentId),
}))
.filter((instance) => instance.instance) as Array<{
schedule: Omit<(typeof schedules)[number], "instances">;
instance: NonNullable<(typeof schedules)[number]["instances"][number]>;
}>;
if (instancesWithSchedule.length === 0) {
this.logger.info("No instances found for environment", {
environmentId,
projectId,
});
return {
recovered: [],
skipped: [],
};
}
const results = {
recovered: [],
skipped: [],
} as { recovered: string[]; skipped: string[] };
for (const { instance, schedule } of instancesWithSchedule) {
this.logger.debug("Recovering schedule", {
schedule,
instance,
});
const [recoverError, result] = await tryCatch(
this.#recoverTaskScheduleInstance({ instance, schedule })
);
if (recoverError) {
this.logger.error("Error recovering schedule", {
error: recoverError instanceof Error ? recoverError.message : String(recoverError),
});
span.setAttribute("recover_error", true);
span.setAttribute(
"recover_error_message",
recoverError instanceof Error ? recoverError.message : String(recoverError)
);
} else {
span.setAttribute("recover_success", true);
if (result === "recovered") {
results.recovered.push(instance.id);
} else {
results.skipped.push(instance.id);
}
}
}
return results;
});
}
async #recoverTaskScheduleInstance({
instance,
schedule,
}: {
instance: {
id: string;
environmentId: string;
createdAt: Date;
};
schedule: { id: string; generatorExpression: string; timezone: string | null };
}) {
// inspect the schedule worker to see if there is a job for this instance
const job = await this.worker.getJob(`scheduled-task-instance:${instance.id}`);
if (job) {
this.logger.debug("Job already exists for instance", {
instanceId: instance.id,
job,
schedule,
});
return "skipped";
}
this.logger.debug("No job found for instance, registering next run", {
instanceId: instance.id,
schedule,
});
// No `lastScheduleTime` passed — `registerNextTaskScheduleInstance`
// will derive it from the cron's previous slot (with a createdAt
// guard) so the post-recovery fire reports an accurate
// `payload.lastTimestamp`.
await this.registerNextTaskScheduleInstance({ instanceId: instance.id });
return "recovered";
}
async getJob(id: string) {
return this.worker.getJob(id);
}
async quit() {
this.logger.info("Shutting down schedule engine");
try {
await this.worker.stop();
this.logger.info("Schedule engine worker stopped successfully");
} catch (error) {
this.logger.error("Error stopping schedule engine worker", {
error: error instanceof Error ? error.message : String(error),
});
throw error;
}
}
}
@@ -0,0 +1,72 @@
import { parseExpression } from "cron-parser";
export function calculateNextScheduledTimestampFromNow(schedule: string, timezone: string | null) {
return calculateNextScheduledTimestamp(schedule, timezone, new Date());
}
export function calculateNextScheduledTimestamp(
schedule: string,
timezone: string | null,
lastScheduledTimestamp: Date = new Date()
) {
const nextStep = calculateNextStep(schedule, timezone, lastScheduledTimestamp);
if (nextStep.getTime() < Date.now()) {
// If the next step is in the past, we just need to calculate the next step from now
return calculateNextStep(schedule, timezone, new Date());
}
return nextStep;
}
function calculateNextStep(schedule: string, timezone: string | null, currentDate: Date) {
return parseExpression(schedule, {
currentDate,
utc: timezone === null,
tz: timezone ?? undefined,
})
.next()
.toDate();
}
/**
* Cron's previous slot relative to `fromTimestamp`. For a continuously-
* running schedule this equals the actual last fire time; for paused or
* DST-edge cases it's an approximation. Used only on the recovery path
* where the actual last fire isn't recoverable from in-flight worker state.
*/
export function previousScheduledTimestamp(
cron: string,
timezone: string | null,
fromTimestamp: Date = new Date()
) {
return parseExpression(cron, {
currentDate: fromTimestamp,
utc: timezone === null,
tz: timezone ?? undefined,
})
.prev()
.toDate();
}
export function nextScheduledTimestamps(
cron: string,
timezone: string | null,
lastScheduledTimestamp: Date,
count: number = 1
) {
const result: Array<Date> = [];
let nextScheduledTimestamp = lastScheduledTimestamp;
for (let i = 0; i < count; i++) {
nextScheduledTimestamp = calculateNextScheduledTimestamp(
cron,
timezone,
nextScheduledTimestamp
);
result.push(nextScheduledTimestamp);
}
return result;
}
@@ -0,0 +1,95 @@
import type { Logger } from "@trigger.dev/core/logger";
import type { Meter, Tracer } from "@internal/tracing";
import type { Prisma, PrismaClient } from "@trigger.dev/database";
import type { RedisOptions } from "@internal/redis";
export type SchedulingEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
include: { project: true; organization: true; orgMember: true };
}>;
export type TriggerScheduledTaskParams = {
taskIdentifier: string;
environment: SchedulingEnvironment;
payload: {
scheduleId: string;
type: "DECLARATIVE" | "IMPERATIVE";
timestamp: Date;
lastTimestamp?: Date;
externalId?: string;
timezone: string;
upcoming: Date[];
};
scheduleInstanceId: string;
scheduleId: string;
exactScheduleTime?: Date;
};
export type TriggerScheduledTaskErrorType = "QUEUE_LIMIT" | "OUT_OF_ENTITLEMENTS" | "SYSTEM_ERROR";
export interface TriggerScheduledTaskCallback {
(params: TriggerScheduledTaskParams): Promise<{
success: boolean;
error?: string;
errorType?: TriggerScheduledTaskErrorType;
}>;
}
export interface ScheduleEngineOptions {
logger?: Logger;
logLevel?: string;
prisma: PrismaClient;
redis: RedisOptions;
worker: {
concurrency: number;
workers?: number;
tasksPerWorker?: number;
pollIntervalMs?: number;
shutdownTimeoutMs?: number;
disabled?: boolean;
};
distributionWindow?: {
seconds: number;
};
tracer?: Tracer;
meter?: Meter;
onTriggerScheduledTask: TriggerScheduledTaskCallback;
isDevEnvironmentConnectedHandler: (environmentId: string) => Promise<boolean>;
onRegisterScheduleInstance?: (instanceId: string) => Promise<void>;
}
export interface UpsertScheduleParams {
projectId: string;
schedule: {
friendlyId?: string;
taskIdentifier: string;
deduplicationKey?: string;
cron: string;
timezone?: string;
externalId?: string;
environments: string[];
};
}
export interface TriggerScheduleParams {
instanceId: string;
finalAttempt: boolean;
exactScheduleTime?: Date;
lastScheduleTime?: Date;
}
export interface RegisterScheduleInstanceParams {
instanceId: string;
/**
* Anchor for computing the next cron slot. Defaults to now() when omitted.
* This advances on every tick (fired or skipped) so the next slot keeps
* marching forward regardless of skip reasons.
*/
fromTimestamp?: Date;
/**
* The actual previous fire time to embed in the next worker job's payload,
* which becomes that job's `payload.lastTimestamp` on dequeue. Distinct
* from `fromTimestamp` so that skipped ticks (inactive schedule, dev env
* disconnected, etc.) do NOT advance this — only real fires do.
*/
lastScheduleTime?: Date;
}
@@ -0,0 +1,19 @@
import { z } from "zod";
export const scheduleWorkerCatalog = {
"schedule.triggerScheduledTask": {
schema: z.object({
instanceId: z.string(),
exactScheduleTime: z.coerce.date(),
// Optional for backward compat with in-flight jobs enqueued by older
// engines. After deploy, every newly-enqueued job populates this with
// the just-fired schedule time so the next dequeue can report
// payload.lastTimestamp accurately without a DB round-trip.
lastScheduleTime: z.coerce.date().optional(),
}),
visibilityTimeoutMs: 60_000,
retry: {
maxAttempts: 5,
},
},
};
@@ -0,0 +1,7 @@
export { ScheduleEngine } from "./engine/index.js";
export type {
ScheduleEngineOptions,
TriggerScheduleParams,
TriggerScheduledTaskCallback,
TriggerScheduledTaskErrorType,
} from "./engine/types.js";
@@ -0,0 +1,242 @@
import { containerTest } from "@internal/testcontainers";
import { trace } from "@internal/tracing";
import { setTimeout } from "timers/promises";
import { describe, expect, vi } from "vitest";
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
import { ScheduleEngine } from "../src/index.js";
describe("ScheduleEngine Integration", () => {
containerTest(
"should process full schedule lifecycle through worker with multiple executions",
{ timeout: 240_000 }, // Increase timeout for multiple executions (4 minutes)
async ({ prisma, redisOptions }) => {
// Real callback function for testing expectations
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
const triggerCalls: Array<{
params: TriggerScheduledTaskParams;
executionTime: Date;
}> = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: false, // Enable worker for full integration test
pollIntervalMs: 100, // Poll frequently for faster test execution
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
const executionTime = new Date(); // Capture when callback is actually called
console.log(
`TriggerScheduledTask called at: ${executionTime.toISOString()} (execution #${
triggerCalls.length + 1
})`
);
console.log("TriggerScheduledTask", params);
triggerCalls.push({ params, executionTime });
return { success: true };
},
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
});
try {
// Create real database records
const organization = await prisma.organization.create({
data: {
title: "Test Organization",
slug: "test-org",
},
});
const project = await prisma.project.create({
data: {
name: "Test Project",
slug: "test-project",
externalRef: "test-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "test-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_test_1234",
pkApiKey: "pk_test_1234",
shortcode: "test-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_abc123",
taskIdentifier: "test-task",
projectId: project.id,
deduplicationKey: "test-dedup",
userProvidedDeduplicationKey: false,
generatorExpression: "* * * * *", // Every minute
generatorDescription: "Every minute",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
externalId: "ext-123",
},
});
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
// Manually enqueue the first scheduled task to kick off the lifecycle.
// Anchor expectations to the first observed `exactScheduleTime` rather
// than a precomputed wall-clock value — registration that happens to
// straddle a minute boundary used to flake tests asserting against a
// pre-baked "next minute".
await engine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
// Wait for the first execution
console.log("Waiting for first execution...");
const startTime = Date.now();
const maxWaitTime = 70_000; // 70 seconds max wait for first execution
while (triggerCalls.length === 0 && Date.now() - startTime < maxWaitTime) {
await setTimeout(100);
}
expect(triggerCalls.length).toBeGreaterThanOrEqual(1);
// Verify the first execution
const firstExecution = triggerCalls[0];
console.log("First execution verified, waiting for second execution...");
// Wait for the second execution (should happen ~1 minute after the first)
const secondExecutionStartTime = Date.now();
const maxWaitForSecond = 80_000; // 80 seconds max wait for second execution
while (
triggerCalls.length < 2 &&
Date.now() - secondExecutionStartTime < maxWaitForSecond
) {
await setTimeout(100);
}
expect(triggerCalls.length).toBeGreaterThanOrEqual(2);
const secondExecution = triggerCalls[1];
console.log("Second execution verified!");
// Give a small delay for database updates to complete before checking
await setTimeout(500);
// Verify both executions have correct timing and distribution window behavior
for (let i = 0; i < 2; i++) {
const execution = triggerCalls[i];
const expectedScheduleTime = execution.params.exactScheduleTime;
if (expectedScheduleTime) {
// Calculate the distribution window (10 seconds before the scheduled time)
const distributionWindowStart = new Date(expectedScheduleTime);
distributionWindowStart.setSeconds(distributionWindowStart.getSeconds() - 10);
console.log(`Execution ${i + 1}:`);
console.log(" Scheduled time:", expectedScheduleTime.toISOString());
console.log(" Distribution window start:", distributionWindowStart.toISOString());
console.log(" Actual execution time:", execution.executionTime.toISOString());
// Verify the callback was executed within the distribution window
expect(execution.executionTime.getTime()).toBeGreaterThanOrEqual(
distributionWindowStart.getTime()
);
expect(execution.executionTime.getTime()).toBeLessThanOrEqual(
expectedScheduleTime.getTime()
);
}
}
// Anchor all expectations to what the engine actually fired with, so
// the test stays deterministic regardless of when within a minute it
// started.
const firstScheduledTime = firstExecution.params.exactScheduleTime;
const secondScheduledTime = secondExecution.params.exactScheduleTime;
expect(firstScheduledTime).toBeDefined();
expect(secondScheduledTime).toBeDefined();
// Each cron slot for "* * * * *" is exactly 60s apart.
expect(secondScheduledTime!.getTime() - firstScheduledTime!.getTime()).toBe(60_000);
// Verify the first execution parameters
expect(firstExecution.params).toEqual({
taskIdentifier: "test-task",
environment: expect.objectContaining({
id: environment.id,
type: "PRODUCTION",
project: expect.objectContaining({
id: project.id,
name: "Test Project",
slug: "test-project",
}),
organization: expect.objectContaining({
id: organization.id,
title: "Test Organization",
slug: "test-org",
}),
}),
payload: {
scheduleId: "sched_abc123",
type: "DECLARATIVE",
timestamp: firstScheduledTime,
// First-ever fire: no `lastScheduleTime` carried in the worker
// payload and `instance.lastScheduledTimestamp` is null on a
// fresh instance, so lastTimestamp is undefined. This preserves
// the `if (!payload.lastTimestamp)` first-run sentinel customers
// rely on.
lastTimestamp: undefined,
externalId: "ext-123",
timezone: "UTC",
upcoming: expect.arrayContaining([expect.any(Date)]),
},
scheduleInstanceId: scheduleInstance.id,
scheduleId: taskSchedule.id,
exactScheduleTime: firstScheduledTime,
});
// Verify the second execution parameters
expect(secondExecution.params).toEqual({
taskIdentifier: "test-task",
environment: expect.objectContaining({
id: environment.id,
type: "PRODUCTION",
}),
payload: {
scheduleId: "sched_abc123",
type: "DECLARATIVE",
timestamp: secondScheduledTime,
// The previous fire's exactScheduleTime is carried through the
// worker payload as `lastScheduleTime` and surfaced here.
lastTimestamp: firstScheduledTime,
externalId: "ext-123",
timezone: "UTC",
upcoming: expect.arrayContaining([expect.any(Date)]),
},
scheduleInstanceId: scheduleInstance.id,
scheduleId: taskSchedule.id,
exactScheduleTime: secondScheduledTime,
});
} finally {
// Clean up: stop the worker
await engine.quit();
}
}
);
});
@@ -0,0 +1,111 @@
import { containerTest } from "@internal/testcontainers";
import { trace } from "@internal/tracing";
import { describe, expect, vi } from "vitest";
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
import { ScheduleEngine } from "../src/index.js";
describe("ScheduleEngine Integration (part 2)", () => {
// Deploy-moment backward compatibility. At deploy time, in-flight Redis jobs
// were enqueued by the old engine — their payload has no `lastScheduleTime`
// field — and `instance.lastScheduledTimestamp` is still populated (last
// written by the old engine pre-deploy). The new engine must report that DB
// value as `payload.lastTimestamp` so customers don't see a transient
// `undefined` for the one fire per schedule that drains the legacy queue.
containerTest(
"should fall back to instance.lastScheduledTimestamp when payload lacks lastScheduleTime",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const triggerCalls: TriggerScheduledTaskParams[] = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: true, // Don't actually run the worker — calling triggerScheduledTask directly
pollIntervalMs: 1000,
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
triggerCalls.push(params);
return { success: true };
},
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
});
try {
const organization = await prisma.organization.create({
data: { title: "Legacy Payload Org", slug: "legacy-payload-org" },
});
const project = await prisma.project.create({
data: {
name: "Legacy Payload Project",
slug: "legacy-payload-project",
externalRef: "legacy-payload-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "legacy-payload-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_legacy_1234",
pkApiKey: "pk_legacy_1234",
shortcode: "legacy-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_legacy_payload",
taskIdentifier: "legacy-payload-task",
projectId: project.id,
deduplicationKey: "legacy-payload-dedup",
userProvidedDeduplicationKey: false,
generatorExpression: "*/5 * * * *",
generatorDescription: "Every 5 minutes",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
externalId: "legacy-ext",
},
});
// Pre-populate lastScheduledTimestamp on the instance — simulates the
// value the old engine wrote to the DB before this PR deployed.
const preDeployLastFire = new Date("2026-04-30T10:00:00.000Z");
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
lastScheduledTimestamp: preDeployLastFire,
},
});
// Call triggerScheduledTask directly without lastScheduleTime,
// simulating an in-flight Redis job enqueued by the old engine.
const exactScheduleTime = new Date("2026-04-30T10:05:00.000Z");
await engine.triggerScheduledTask({
instanceId: scheduleInstance.id,
finalAttempt: false,
exactScheduleTime,
// lastScheduleTime intentionally omitted — legacy payload shape
});
expect(triggerCalls.length).toBe(1);
expect(triggerCalls[0].payload.timestamp).toEqual(exactScheduleTime);
// Falls back to instance.lastScheduledTimestamp from the DB rather
// than reporting undefined for this one transitional fire.
expect(triggerCalls[0].payload.lastTimestamp).toEqual(preDeployLastFire);
} finally {
await engine.quit();
}
}
);
});
@@ -0,0 +1,580 @@
import { containerTest } from "@internal/testcontainers";
import { trace } from "@internal/tracing";
import { describe, expect, vi } from "vitest";
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
import { ScheduleEngine } from "../src/index.js";
describe("Schedule Recovery", () => {
containerTest(
"should recover schedules when no existing jobs are found",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
const triggerCalls: TriggerScheduledTaskParams[] = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: true, // Disable worker to prevent automatic execution
pollIntervalMs: 1000,
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
triggerCalls.push(params);
return { success: true };
},
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
});
try {
// Create test data
const organization = await prisma.organization.create({
data: {
title: "Recovery Test Org",
slug: "recovery-test-org",
},
});
const project = await prisma.project.create({
data: {
name: "Recovery Test Project",
slug: "recovery-test-project",
externalRef: "recovery-test-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "recovery-test-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_recovery_test_1234",
pkApiKey: "pk_recovery_test_1234",
shortcode: "recovery-test-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_recovery_123",
taskIdentifier: "recovery-test-task",
projectId: project.id,
deduplicationKey: "recovery-test-dedup",
userProvidedDeduplicationKey: false,
generatorExpression: "0 */5 * * *", // Every 5 minutes
generatorDescription: "Every 5 minutes",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
externalId: "recovery-ext-123",
},
});
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
// Verify no job exists initially
const jobBeforeRecovery = await engine.getJob(
`scheduled-task-instance:${scheduleInstance.id}`
);
expect(jobBeforeRecovery).toBeNull();
// Perform recovery
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
// Verify that a job was created. The engine no longer persists
// nextScheduledTimestamp; correctness is now determined entirely by the
// job sitting in the worker queue.
const jobAfterRecovery = await engine.getJob(
`scheduled-task-instance:${scheduleInstance.id}`
);
expect(jobAfterRecovery).not.toBeNull();
expect(jobAfterRecovery?.job).toBe("schedule.triggerScheduledTask");
} finally {
await engine.quit();
}
}
);
containerTest(
"should not create duplicate jobs when schedule already has an active job",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
const triggerCalls: TriggerScheduledTaskParams[] = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: true, // Disable worker to prevent automatic execution
pollIntervalMs: 1000,
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
triggerCalls.push(params);
return { success: true };
},
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
});
try {
// Create test data
const organization = await prisma.organization.create({
data: {
title: "Duplicate Test Org",
slug: "duplicate-test-org",
},
});
const project = await prisma.project.create({
data: {
name: "Duplicate Test Project",
slug: "duplicate-test-project",
externalRef: "duplicate-test-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "duplicate-test-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_duplicate_test_1234",
pkApiKey: "pk_duplicate_test_1234",
shortcode: "duplicate-test-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_duplicate_123",
taskIdentifier: "duplicate-test-task",
projectId: project.id,
deduplicationKey: "duplicate-test-dedup",
userProvidedDeduplicationKey: false,
generatorExpression: "0 */10 * * *", // Every 10 minutes
generatorDescription: "Every 10 minutes",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
externalId: "duplicate-ext-123",
},
});
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
// First, register the schedule normally
await engine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
// Verify job exists
const jobAfterFirstRegistration = await engine.getJob(
`scheduled-task-instance:${scheduleInstance.id}`
);
expect(jobAfterFirstRegistration).not.toBeNull();
const firstJobId = jobAfterFirstRegistration?.id;
// Now run recovery - it should not create a duplicate job
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
// Verify the same job still exists (no duplicate created)
const jobAfterRecovery = await engine.getJob(
`scheduled-task-instance:${scheduleInstance.id}`
);
expect(jobAfterRecovery).not.toBeNull();
expect(jobAfterRecovery?.id).toBe(firstJobId);
expect(jobAfterRecovery?.deduplicationKey).toBe(jobAfterFirstRegistration.deduplicationKey);
} finally {
await engine.quit();
}
}
);
containerTest(
"should recover multiple schedules in the same environment",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
const triggerCalls: TriggerScheduledTaskParams[] = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: true, // Disable worker to prevent automatic execution
pollIntervalMs: 1000,
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
triggerCalls.push(params);
return { success: true };
},
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
});
try {
// Create test data
const organization = await prisma.organization.create({
data: {
title: "Multiple Test Org",
slug: "multiple-test-org",
},
});
const project = await prisma.project.create({
data: {
name: "Multiple Test Project",
slug: "multiple-test-project",
externalRef: "multiple-test-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "multiple-test-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_multiple_test_1234",
pkApiKey: "pk_multiple_test_1234",
shortcode: "multiple-test-short",
},
});
// Create multiple task schedules
const schedules = [];
const instances = [];
for (let i = 1; i <= 3; i++) {
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: `sched_multiple_${i}`,
taskIdentifier: `multiple-test-task-${i}`,
projectId: project.id,
deduplicationKey: `multiple-test-dedup-${i}`,
userProvidedDeduplicationKey: false,
generatorExpression: `${i} */15 * * *`, // Every 15 minutes at different minute offsets
generatorDescription: `Every 15 minutes (${i})`,
timezone: "UTC",
type: "DECLARATIVE",
active: true,
externalId: `multiple-ext-${i}`,
},
});
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
schedules.push(taskSchedule);
instances.push(scheduleInstance);
}
// Verify no jobs exist initially
for (const instance of instances) {
const job = await engine.getJob(`scheduled-task-instance:${instance.id}`);
expect(job).toBeNull();
}
// Perform recovery
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
// Verify that jobs were created for all instances. The engine no longer
// persists nextScheduledTimestamp — the worker-queue presence is the
// source of truth.
for (const instance of instances) {
const job = await engine.getJob(`scheduled-task-instance:${instance.id}`);
expect(job).not.toBeNull();
expect(job?.job).toBe("schedule.triggerScheduledTask");
}
} finally {
await engine.quit();
}
}
);
containerTest(
"should handle recovery gracefully when no schedules exist in environment",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
const triggerCalls: TriggerScheduledTaskParams[] = [];
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: {
concurrency: 1,
disabled: true, // Disable worker to prevent automatic execution
pollIntervalMs: 1000,
},
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async (params) => {
triggerCalls.push(params);
return { success: true };
},
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
});
try {
// Create test data but no schedules
const organization = await prisma.organization.create({
data: {
title: "Empty Test Org",
slug: "empty-test-org",
},
});
const project = await prisma.project.create({
data: {
name: "Empty Test Project",
slug: "empty-test-project",
externalRef: "empty-test-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "empty-test-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_empty_test_1234",
pkApiKey: "pk_empty_test_1234",
shortcode: "empty-test-short",
},
});
// Perform recovery on empty environment - should not throw errors
await expect(
engine.recoverSchedulesInEnvironment(project.id, environment.id)
).resolves.not.toThrow();
} finally {
await engine.quit();
}
}
);
// External-caller backward-compat. Deploy sync (`syncDeclarativeSchedules`)
// and schedule upsert both call `registerNextTaskScheduleInstance` with no
// `lastScheduleTime`. They run on every app deploy and on every cron edit.
// For an existing-and-firing schedule, the call must NOT clobber the
// worker payload's `lastScheduleTime` with `undefined` — otherwise the
// next fire would surface a stale frozen DB-column value to the customer
// (since this PR stops writing that column). The function must derive a
// sensible `lastScheduleTime` from the cron expression's previous slot
// when the caller doesn't pass one.
containerTest(
"should derive lastScheduleTime from cron when external callers omit it",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async () => ({ success: true }),
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
});
try {
const organization = await prisma.organization.create({
data: { title: "External Caller Org", slug: "external-caller-org" },
});
const project = await prisma.project.create({
data: {
name: "External Caller Project",
slug: "external-caller-project",
externalRef: "external-caller-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "external-caller-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_external_1234",
pkApiKey: "pk_external_1234",
shortcode: "external-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_external_caller",
taskIdentifier: "external-caller-task",
projectId: project.id,
deduplicationKey: "external-caller-dedup",
userProvidedDeduplicationKey: false,
generatorExpression: "*/5 * * * *",
generatorDescription: "Every 5 minutes",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
},
});
// Backdate the instance so the cron's previous slot postdates
// createdAt — this simulates a long-running schedule, the case
// Devin flagged (deploy clobbers lastScheduleTime, post-deploy fire
// would otherwise read from a frozen DB column).
const longAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
await prisma.taskScheduleInstance.update({
where: { id: scheduleInstance.id },
data: { createdAt: longAgo },
});
// External-caller pattern — no lastScheduleTime.
await engine.registerNextTaskScheduleInstance({
instanceId: scheduleInstance.id,
});
const job = await engine.getJob(`scheduled-task-instance:${scheduleInstance.id}`);
expect(job).not.toBeNull();
// The function should have derived lastScheduleTime from cron,
// putting a real timestamp into the worker payload rather than
// undefined. The Redis worker stores payloads as JSON, so the value
// is a string when read back here — Zod re-coerces it to Date on
// dequeue (workerCatalog uses `z.coerce.date()`).
const enqueuedLastScheduleTime = (job!.item as { lastScheduleTime?: string })
.lastScheduleTime;
expect(enqueuedLastScheduleTime).toBeDefined();
const derived = new Date(enqueuedLastScheduleTime!);
// The derived value should match the cron's previous slot — for
// `*/5 * * * *`, a 5-minute boundary in the recent past.
expect(derived.getTime()).toBeLessThan(Date.now());
expect(derived.getUTCSeconds()).toBe(0);
expect(derived.getUTCMinutes() % 5).toBe(0);
} finally {
await engine.quit();
}
}
);
// Brand-new schedules must NOT receive a cron-derived lastScheduleTime —
// the cron's previous slot predates the instance, so it's not a real
// previous fire. The first-run sentinel (`if (!payload.lastTimestamp)`)
// must keep working.
containerTest(
"should leave lastScheduleTime undefined for brand-new schedules",
{ timeout: 30_000 },
async ({ prisma, redisOptions }) => {
const engine = new ScheduleEngine({
prisma,
redis: redisOptions,
distributionWindow: { seconds: 10 },
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
tracer: trace.getTracer("test", "0.0.0"),
onTriggerScheduledTask: async () => ({ success: true }),
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
});
try {
const organization = await prisma.organization.create({
data: { title: "Brand New Org", slug: "brand-new-org" },
});
const project = await prisma.project.create({
data: {
name: "Brand New Project",
slug: "brand-new-project",
externalRef: "brand-new-ref",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: "brand-new-env",
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_brandnew_1234",
pkApiKey: "pk_brandnew_1234",
shortcode: "brandnew-short",
},
});
const taskSchedule = await prisma.taskSchedule.create({
data: {
friendlyId: "sched_brand_new",
taskIdentifier: "brand-new-task",
projectId: project.id,
deduplicationKey: "brand-new-dedup",
userProvidedDeduplicationKey: false,
// Hourly cron — the previous slot is plausibly minutes-to-an-hour
// ago, comfortably predating an instance just created.
generatorExpression: "0 * * * *",
generatorDescription: "Hourly",
timezone: "UTC",
type: "DECLARATIVE",
active: true,
},
});
const scheduleInstance = await prisma.taskScheduleInstance.create({
data: {
taskScheduleId: taskSchedule.id,
environmentId: environment.id,
projectId: project.id,
active: true,
},
});
await engine.registerNextTaskScheduleInstance({
instanceId: scheduleInstance.id,
});
const job = await engine.getJob(`scheduled-task-instance:${scheduleInstance.id}`);
expect(job).not.toBeNull();
const enqueuedLastScheduleTime = (job!.item as { lastScheduleTime?: Date })
.lastScheduleTime;
// Brand-new schedule: cron's previous slot predates instance.createdAt,
// so the function leaves lastScheduleTime undefined — the first fire
// will report `payload.lastTimestamp: undefined` and customer first-run
// sentinel patterns keep working.
expect(enqueuedLastScheduleTime).toBeUndefined();
} finally {
await engine.quit();
}
}
);
});
@@ -0,0 +1,4 @@
import { vi } from "vitest";
// Set extended timeout for container tests
vi.setConfig({ testTimeout: 60_000 });
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"outDir": "dist",
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"declaration": true
}
}
@@ -0,0 +1,8 @@
{
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
"compilerOptions": {
"moduleResolution": "Node16",
"module": "Node16",
"customConditions": ["@triggerdotdev/source"]
}
}
@@ -0,0 +1,20 @@
{
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.test.ts"],
"references": [{ "path": "./tsconfig.src.json" }],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"types": ["vitest/globals"],
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
@@ -0,0 +1,18 @@
import { defineConfig } from "vitest/config";
import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
export default defineConfig({
test: {
sequence: { sequencer: DurationShardingSequencer },
globals: true,
// CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
retry: process.env.CI ? 2 : 0,
environment: "node",
setupFiles: ["./test/setup.ts"],
testTimeout: 30000,
hookTimeout: 30000,
},
esbuild: {
target: "node18",
},
});