chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
# @internal/schedule-engine
|
||||
|
||||
The `@internal/schedule-engine` package encapsulates all scheduling logic for Trigger.dev, providing a clean API boundary for managing scheduled tasks and their execution.
|
||||
|
||||
## Architecture
|
||||
|
||||
The ScheduleEngine follows the same pattern as the RunEngine, providing:
|
||||
|
||||
- **Centralized Schedule Management**: All schedule-related operations go through the ScheduleEngine
|
||||
- **Redis Worker Integration**: Built-in Redis-based distributed task scheduling
|
||||
- **Distributed Execution**: Prevents thundering herd issues by distributing executions across time windows
|
||||
- **Comprehensive Testing**: Built-in utilities for testing schedule behavior
|
||||
|
||||
## Key Components
|
||||
|
||||
### ScheduleEngine Class
|
||||
|
||||
The main interface for all schedule operations:
|
||||
|
||||
```typescript
|
||||
import { ScheduleEngine } from "@internal/schedule-engine";
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: {
|
||||
/* Redis configuration */
|
||||
},
|
||||
worker: {
|
||||
/* Worker configuration */
|
||||
},
|
||||
distributionWindow: { seconds: 30 }, // Optional: default 30s
|
||||
});
|
||||
|
||||
// Register next schedule instance
|
||||
await engine.registerNextTaskScheduleInstance({ instanceId });
|
||||
|
||||
// Upsert a schedule
|
||||
await engine.upsertTaskSchedule({
|
||||
projectId,
|
||||
schedule: {
|
||||
taskIdentifier: "my-task",
|
||||
cron: "0 */5 * * *",
|
||||
timezone: "UTC",
|
||||
environments: ["env-1", "env-2"],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Distributed Scheduling
|
||||
|
||||
The engine includes built-in distributed scheduling to prevent all scheduled tasks from executing at exactly the same moment:
|
||||
|
||||
```typescript
|
||||
import { calculateDistributedExecutionTime } from "@internal/schedule-engine";
|
||||
|
||||
const exactTime = new Date("2024-01-01T12:00:00Z");
|
||||
const distributedTime = calculateDistributedExecutionTime(exactTime, 30); // 30-second window
|
||||
```
|
||||
|
||||
### Schedule Calculation
|
||||
|
||||
High-performance CRON schedule calculation with optimization for old timestamps:
|
||||
|
||||
```typescript
|
||||
import {
|
||||
calculateNextScheduledTimestampFromNow,
|
||||
nextScheduledTimestamps,
|
||||
} from "@internal/schedule-engine";
|
||||
|
||||
const nextRun = calculateNextScheduledTimestampFromNow("0 */5 * * *", "UTC");
|
||||
const upcoming = nextScheduledTimestamps("0 */5 * * *", "UTC", nextRun, 5);
|
||||
```
|
||||
|
||||
## Integration with Webapp
|
||||
|
||||
The ScheduleEngine should be the **API boundary** between the webapp and schedule logic. Services in the webapp should call into the ScheduleEngine rather than implementing schedule logic directly.
|
||||
|
||||
### Migration Path
|
||||
|
||||
Currently, the webapp uses individual services like:
|
||||
|
||||
- `RegisterNextTaskScheduleInstanceService`
|
||||
- `TriggerScheduledTaskService`
|
||||
- Schedule calculation utilities
|
||||
|
||||
These should be replaced with ScheduleEngine method calls:
|
||||
|
||||
```typescript
|
||||
// Old approach
|
||||
const service = new RegisterNextTaskScheduleInstanceService(tx);
|
||||
await service.call(instanceId);
|
||||
|
||||
// New approach
|
||||
await scheduleEngine.registerNextTaskScheduleInstance({ instanceId });
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The ScheduleEngine expects these configuration options:
|
||||
|
||||
- `prisma`: PrismaClient instance
|
||||
- `redis`: Redis connection configuration
|
||||
- `worker`: Worker configuration (concurrency, polling intervals)
|
||||
- `distributionWindow`: Optional time window for distributed execution
|
||||
- `tracer`: Optional OpenTelemetry tracer
|
||||
- `meter`: Optional OpenTelemetry meter
|
||||
|
||||
## Testing
|
||||
|
||||
The package includes comprehensive test utilities and examples. See the test directory for usage examples.
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "@internal/schedule-engine",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"@triggerdotdev/source": "./src/index.ts",
|
||||
"import": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"default": "./dist/src/index.js"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@internal/redis": "workspace:*",
|
||||
"@trigger.dev/redis-worker": "workspace:*",
|
||||
"@internal/tracing": "workspace:*",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"@trigger.dev/database": "workspace:*",
|
||||
"cron-parser": "^4.9.0",
|
||||
"cronstrue": "^2.50.0",
|
||||
"nanoid": "3.3.8",
|
||||
"zod": "3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internal/testcontainers": "workspace:*",
|
||||
"rimraf": "6.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.build.json",
|
||||
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
|
||||
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled",
|
||||
"build": "pnpm run clean && tsc -p tsconfig.build.json",
|
||||
"dev": "tsc --watch -p tsconfig.build.json"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Calculates a distributed execution time for a scheduled task.
|
||||
* Tasks are distributed across a time window before the exact schedule time
|
||||
* to prevent thundering herd issues while maintaining schedule accuracy.
|
||||
*/
|
||||
export function calculateDistributedExecutionTime(
|
||||
exactScheduleTime: Date,
|
||||
distributionWindowSeconds: number = 30,
|
||||
instanceId?: string
|
||||
): Date {
|
||||
// Create seed by combining ISO timestamp with optional instanceId
|
||||
// This ensures different instances get different distributions even with same schedule time
|
||||
const timeSeed = exactScheduleTime.toISOString();
|
||||
const seed = instanceId ? `${timeSeed}:${instanceId}` : timeSeed;
|
||||
|
||||
// Use a better hash function (FNV-1a variant) for more uniform distribution
|
||||
let hash = 2166136261; // FNV offset basis (32-bit)
|
||||
|
||||
for (let i = 0; i < seed.length; i++) {
|
||||
hash ^= seed.charCodeAt(i);
|
||||
hash *= 16777619; // FNV prime (32-bit)
|
||||
// Keep it as 32-bit unsigned integer
|
||||
hash = hash >>> 0;
|
||||
}
|
||||
|
||||
// Convert hash to a value between 0 and 1 using better normalization
|
||||
// Use the full 32-bit range for better distribution
|
||||
const normalized = hash / 0xffffffff;
|
||||
|
||||
// Calculate offset in milliseconds (0 to distributionWindowSeconds * 1000)
|
||||
const offsetMs = Math.floor(normalized * distributionWindowSeconds * 1000);
|
||||
|
||||
// Return time that's offsetMs before the exact schedule time
|
||||
return new Date(exactScheduleTime.getTime() - offsetMs);
|
||||
}
|
||||
@@ -0,0 +1,867 @@
|
||||
import type { Counter, Histogram, Meter, Tracer } from "@internal/tracing";
|
||||
import { getMeter, getTracer, startSpan } from "@internal/tracing";
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { Worker, type JobHandlerParams } from "@trigger.dev/redis-worker";
|
||||
import { calculateDistributedExecutionTime } from "./distributedScheduling.js";
|
||||
import {
|
||||
calculateNextScheduledTimestamp,
|
||||
nextScheduledTimestamps,
|
||||
previousScheduledTimestamp,
|
||||
} from "./scheduleCalculation.js";
|
||||
import type {
|
||||
RegisterScheduleInstanceParams,
|
||||
ScheduleEngineOptions,
|
||||
TriggerScheduledTaskCallback,
|
||||
TriggerScheduleParams,
|
||||
} from "./types.js";
|
||||
import { scheduleWorkerCatalog } from "./workerCatalog.js";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
|
||||
export class ScheduleEngine {
|
||||
private worker: Worker<typeof scheduleWorkerCatalog>;
|
||||
private logger: Logger;
|
||||
private tracer: Tracer;
|
||||
private meter: Meter;
|
||||
private distributionWindowSeconds: number;
|
||||
|
||||
// Metrics
|
||||
private scheduleRegistrationCounter: Counter;
|
||||
private scheduleExecutionCounter: Counter;
|
||||
private scheduleExecutionDuration: Histogram;
|
||||
private scheduleExecutionFailureCounter: Counter;
|
||||
private distributionOffsetHistogram: Histogram;
|
||||
private devEnvironmentCheckCounter: Counter;
|
||||
|
||||
prisma: PrismaClient;
|
||||
|
||||
private onTriggerScheduledTask: TriggerScheduledTaskCallback;
|
||||
|
||||
constructor(private readonly options: ScheduleEngineOptions) {
|
||||
this.logger =
|
||||
options.logger ?? new Logger("ScheduleEngine", (this.options.logLevel ?? "info") as any);
|
||||
this.prisma = options.prisma;
|
||||
this.distributionWindowSeconds = options.distributionWindow?.seconds ?? 30;
|
||||
this.onTriggerScheduledTask = options.onTriggerScheduledTask;
|
||||
|
||||
this.tracer = options.tracer ?? getTracer("schedule-engine");
|
||||
this.meter = options.meter ?? getMeter("schedule-engine");
|
||||
|
||||
// Initialize metrics
|
||||
this.scheduleRegistrationCounter = this.meter.createCounter("schedule_registrations_total", {
|
||||
description: "Total number of schedule registrations",
|
||||
});
|
||||
|
||||
this.scheduleExecutionCounter = this.meter.createCounter("schedule_executions_total", {
|
||||
description: "Total number of schedule executions",
|
||||
});
|
||||
|
||||
this.scheduleExecutionDuration = this.meter.createHistogram("schedule_execution_duration_ms", {
|
||||
description: "Duration of schedule execution in milliseconds",
|
||||
unit: "ms",
|
||||
});
|
||||
|
||||
this.scheduleExecutionFailureCounter = this.meter.createCounter(
|
||||
"schedule_execution_failures_total",
|
||||
{
|
||||
description: "Total number of schedule execution failures",
|
||||
}
|
||||
);
|
||||
|
||||
this.distributionOffsetHistogram = this.meter.createHistogram(
|
||||
"schedule_distribution_offset_ms",
|
||||
{
|
||||
description: "Distribution offset from exact schedule time in milliseconds",
|
||||
unit: "ms",
|
||||
}
|
||||
);
|
||||
|
||||
this.devEnvironmentCheckCounter = this.meter.createCounter("dev_environment_checks_total", {
|
||||
description: "Total number of development environment connectivity checks",
|
||||
});
|
||||
|
||||
this.worker = new Worker({
|
||||
name: "schedule-engine-worker",
|
||||
redisOptions: {
|
||||
...options.redis,
|
||||
keyPrefix: `${options.redis.keyPrefix ?? ""}schedule:`,
|
||||
},
|
||||
catalog: scheduleWorkerCatalog,
|
||||
concurrency: {
|
||||
limit: options.worker.concurrency,
|
||||
workers: options.worker.workers,
|
||||
tasksPerWorker: options.worker.tasksPerWorker,
|
||||
},
|
||||
pollIntervalMs: options.worker.pollIntervalMs,
|
||||
shutdownTimeoutMs: options.worker.shutdownTimeoutMs,
|
||||
logger: new Logger("ScheduleEngineWorker", (options.logLevel ?? "info") as any),
|
||||
jobs: {
|
||||
"schedule.triggerScheduledTask": this.#handleTriggerScheduledTaskJob.bind(this),
|
||||
},
|
||||
});
|
||||
|
||||
if (!options.worker.disabled) {
|
||||
this.worker.start();
|
||||
this.logger.info("Schedule engine worker started", {
|
||||
concurrency: options.worker.concurrency,
|
||||
pollIntervalMs: options.worker.pollIntervalMs,
|
||||
distributionWindowSeconds: this.distributionWindowSeconds,
|
||||
});
|
||||
} else {
|
||||
this.logger.info("Schedule engine worker disabled");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers the next scheduled instance for a schedule
|
||||
*/
|
||||
async registerNextTaskScheduleInstance(params: RegisterScheduleInstanceParams) {
|
||||
return startSpan(this.tracer, "registerNextTaskScheduleInstance", async (span) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (this.options.onRegisterScheduleInstance) {
|
||||
const [registerError] = await tryCatch(
|
||||
this.options.onRegisterScheduleInstance(params.instanceId)
|
||||
);
|
||||
|
||||
if (registerError) {
|
||||
this.logger.error("Error calling the onRegisterScheduleInstance callback", {
|
||||
instanceId: params.instanceId,
|
||||
error: registerError,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
span.setAttribute("instanceId", params.instanceId);
|
||||
|
||||
this.logger.debug("Starting schedule registration", {
|
||||
instanceId: params.instanceId,
|
||||
});
|
||||
|
||||
try {
|
||||
const instance = await this.prisma.taskScheduleInstance.findFirst({
|
||||
where: {
|
||||
id: params.instanceId,
|
||||
},
|
||||
include: {
|
||||
taskSchedule: true,
|
||||
environment: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!instance) {
|
||||
this.logger.warn("Schedule instance not found during registration", {
|
||||
instanceId: params.instanceId,
|
||||
});
|
||||
span.setAttribute("error", "instance_not_found");
|
||||
return;
|
||||
}
|
||||
|
||||
span.setAttribute("task_schedule_id", instance.taskSchedule.id);
|
||||
span.setAttribute("task_schedule_instance_id", instance.id);
|
||||
span.setAttribute("task_identifier", instance.taskSchedule.taskIdentifier);
|
||||
span.setAttribute("environment_type", instance.environment.type);
|
||||
span.setAttribute("schedule_active", instance.active);
|
||||
span.setAttribute("task_schedule_active", instance.taskSchedule.active);
|
||||
span.setAttribute(
|
||||
"task_schedule_generator_expression",
|
||||
instance.taskSchedule.generatorExpression
|
||||
);
|
||||
|
||||
const fromTimestamp = params.fromTimestamp ?? new Date();
|
||||
span.setAttribute("from_timestamp", fromTimestamp.toISOString());
|
||||
|
||||
const nextScheduledTimestamp = calculateNextScheduledTimestamp(
|
||||
instance.taskSchedule.generatorExpression,
|
||||
instance.taskSchedule.timezone,
|
||||
fromTimestamp
|
||||
);
|
||||
|
||||
span.setAttribute("next_scheduled_timestamp", nextScheduledTimestamp.toISOString());
|
||||
|
||||
const schedulingDelayMs = nextScheduledTimestamp.getTime() - Date.now();
|
||||
span.setAttribute("scheduling_delay_ms", schedulingDelayMs);
|
||||
|
||||
this.logger.debug("Calculated next schedule timestamp", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
nextScheduledTimestamp: nextScheduledTimestamp.toISOString(),
|
||||
schedulingDelayMs,
|
||||
generatorExpression: instance.taskSchedule.generatorExpression,
|
||||
timezone: instance.taskSchedule.timezone,
|
||||
});
|
||||
|
||||
// Determine the lastScheduleTime to embed in the next worker job's
|
||||
// payload. If the caller passed it explicitly (the after-fire path
|
||||
// does this with the just-fired timestamp, the after-skip path
|
||||
// carries the existing value forward), use that. Otherwise — every
|
||||
// external caller (deploy sync, schedule upsert, recovery) — derive
|
||||
// from the cron expression's previous slot.
|
||||
//
|
||||
// Without this fallback, every deploy / cron edit would clobber the
|
||||
// existing in-flight job's lastScheduleTime with `undefined`, and
|
||||
// the next fire would surface a frozen DB-column value to the
|
||||
// customer (since this PR stops writing that column). Pure cron
|
||||
// math, no DB read on top of the existing instance load — the
|
||||
// recovery loop already pays the cost of loading the instance.
|
||||
let lastScheduleTime = params.lastScheduleTime;
|
||||
if (lastScheduleTime === undefined) {
|
||||
try {
|
||||
const cronPrev = previousScheduledTimestamp(
|
||||
instance.taskSchedule.generatorExpression,
|
||||
instance.taskSchedule.timezone
|
||||
);
|
||||
// Guarded against the cron's previous slot predating the
|
||||
// instance itself — for a brand-new schedule, the slot is from
|
||||
// before the schedule existed, so `undefined` is the honest
|
||||
// answer (preserves the `if (!payload.lastTimestamp)` first-run
|
||||
// sentinel customers rely on).
|
||||
if (cronPrev.getTime() > instance.createdAt.getTime()) {
|
||||
lastScheduleTime = cronPrev;
|
||||
}
|
||||
} catch {
|
||||
// Malformed cron — leave undefined.
|
||||
}
|
||||
}
|
||||
|
||||
await this.enqueueScheduledTask(
|
||||
params.instanceId,
|
||||
nextScheduledTimestamp,
|
||||
lastScheduleTime
|
||||
);
|
||||
|
||||
// Record metrics
|
||||
this.scheduleRegistrationCounter.add(1, {
|
||||
environment_type: instance.environment.type,
|
||||
schedule_type: instance.taskSchedule.type,
|
||||
});
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
this.logger.debug("Schedule registration completed", {
|
||||
instanceId: params.instanceId,
|
||||
durationMs: duration,
|
||||
});
|
||||
|
||||
span.setAttribute("success", true);
|
||||
span.setAttribute("duration_ms", duration);
|
||||
} catch (error) {
|
||||
const duration = Date.now() - startTime;
|
||||
this.logger.error("Failed to register schedule instance", {
|
||||
instanceId: params.instanceId,
|
||||
durationMs: duration,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
|
||||
span.setAttribute("error", true);
|
||||
span.setAttribute("error_message", error instanceof Error ? error.message : String(error));
|
||||
span.setAttribute("duration_ms", duration);
|
||||
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async #handleTriggerScheduledTaskJob({
|
||||
payload,
|
||||
}: JobHandlerParams<typeof scheduleWorkerCatalog, "schedule.triggerScheduledTask">) {
|
||||
await this.triggerScheduledTask({
|
||||
instanceId: payload.instanceId,
|
||||
finalAttempt: false, // TODO: implement retry logic
|
||||
exactScheduleTime: payload.exactScheduleTime,
|
||||
lastScheduleTime: payload.lastScheduleTime,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers a scheduled task (called by the Redis worker)
|
||||
*/
|
||||
async triggerScheduledTask(params: TriggerScheduleParams) {
|
||||
return startSpan(this.tracer, "triggerScheduledTask", async (span) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
span.setAttribute("instanceId", params.instanceId);
|
||||
span.setAttribute("finalAttempt", params.finalAttempt);
|
||||
if (params.exactScheduleTime) {
|
||||
span.setAttribute("exactScheduleTime", params.exactScheduleTime.toISOString());
|
||||
}
|
||||
|
||||
this.logger.debug("Starting scheduled task trigger", {
|
||||
instanceId: params.instanceId,
|
||||
finalAttempt: params.finalAttempt,
|
||||
exactScheduleTime: params.exactScheduleTime?.toISOString(),
|
||||
});
|
||||
|
||||
let taskIdentifier: string | undefined;
|
||||
let environmentType: string | undefined;
|
||||
let scheduleType: string | undefined;
|
||||
|
||||
try {
|
||||
const instance = await this.prisma.taskScheduleInstance.findFirst({
|
||||
where: {
|
||||
id: params.instanceId,
|
||||
},
|
||||
include: {
|
||||
taskSchedule: true,
|
||||
environment: {
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
orgMember: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!instance) {
|
||||
this.logger.debug("Schedule instance not found", {
|
||||
instanceId: params.instanceId,
|
||||
});
|
||||
span.setAttribute("error", "instance_not_found");
|
||||
return;
|
||||
}
|
||||
|
||||
taskIdentifier = instance.taskSchedule.taskIdentifier;
|
||||
environmentType = instance.environment.type;
|
||||
scheduleType = instance.taskSchedule.type;
|
||||
|
||||
span.setAttribute("task_identifier", taskIdentifier);
|
||||
span.setAttribute("environment_type", environmentType);
|
||||
span.setAttribute("schedule_type", scheduleType);
|
||||
span.setAttribute("organization_id", instance.environment.organization.id);
|
||||
span.setAttribute("project_id", instance.environment.project.id);
|
||||
span.setAttribute("environment_id", instance.environment.id);
|
||||
|
||||
// Check if organization/project/environment is still valid
|
||||
if (instance.environment.organization.deletedAt) {
|
||||
this.logger.debug("Organization is deleted, skipping schedule", {
|
||||
instanceId: params.instanceId,
|
||||
scheduleId: instance.taskSchedule.friendlyId,
|
||||
organizationId: instance.environment.organization.id,
|
||||
});
|
||||
span.setAttribute("skip_reason", "organization_deleted");
|
||||
return;
|
||||
}
|
||||
|
||||
if (instance.environment.project.deletedAt) {
|
||||
this.logger.debug("Project is deleted, skipping schedule", {
|
||||
instanceId: params.instanceId,
|
||||
scheduleId: instance.taskSchedule.friendlyId,
|
||||
projectId: instance.environment.project.id,
|
||||
});
|
||||
span.setAttribute("skip_reason", "project_deleted");
|
||||
return;
|
||||
}
|
||||
|
||||
if (instance.environment.archivedAt) {
|
||||
this.logger.debug("Environment is archived, skipping schedule", {
|
||||
instanceId: params.instanceId,
|
||||
scheduleId: instance.taskSchedule.friendlyId,
|
||||
environmentId: instance.environment.id,
|
||||
});
|
||||
span.setAttribute("skip_reason", "environment_archived");
|
||||
return;
|
||||
}
|
||||
|
||||
let shouldTrigger = true;
|
||||
let skipReason: string | undefined;
|
||||
|
||||
if (!instance.active || !instance.taskSchedule.active) {
|
||||
this.logger.debug("Schedule is inactive", {
|
||||
instanceId: params.instanceId,
|
||||
instanceActive: instance.active,
|
||||
scheduleActive: instance.taskSchedule.active,
|
||||
});
|
||||
shouldTrigger = false;
|
||||
skipReason = "schedule_inactive";
|
||||
}
|
||||
|
||||
// For development environments, check if there's an active session
|
||||
if (instance.environment.type === "DEVELOPMENT") {
|
||||
this.devEnvironmentCheckCounter.add(1, {
|
||||
environment_id: instance.environment.id,
|
||||
});
|
||||
|
||||
const [devConnectedError, isConnected] = await tryCatch(
|
||||
this.options.isDevEnvironmentConnectedHandler(instance.environment.id)
|
||||
);
|
||||
|
||||
if (devConnectedError) {
|
||||
this.logger.error("Error checking if development environment is connected", {
|
||||
instanceId: params.instanceId,
|
||||
environmentId: instance.environment.id,
|
||||
error: devConnectedError,
|
||||
});
|
||||
span.setAttribute("dev_connection_check_error", true);
|
||||
shouldTrigger = false;
|
||||
skipReason = "dev_connection_check_failed";
|
||||
} else if (!isConnected) {
|
||||
this.logger.debug("Development environment is disconnected", {
|
||||
instanceId: params.instanceId,
|
||||
environmentId: instance.environment.id,
|
||||
});
|
||||
span.setAttribute("dev_connected", false);
|
||||
shouldTrigger = false;
|
||||
skipReason = "dev_disconnected";
|
||||
} else {
|
||||
span.setAttribute("dev_connected", true);
|
||||
}
|
||||
}
|
||||
|
||||
span.setAttribute("should_trigger", shouldTrigger);
|
||||
if (skipReason) {
|
||||
span.setAttribute("skip_reason", skipReason);
|
||||
}
|
||||
|
||||
// Calculate the schedule timestamp that will be used (regardless of whether we trigger or not)
|
||||
const scheduleTimestamp = params.exactScheduleTime ?? new Date();
|
||||
|
||||
if (shouldTrigger) {
|
||||
// payload.lastTimestamp is the actual previous fire time. Sources, in
|
||||
// order:
|
||||
// 1. params.lastScheduleTime — populated by the engine when this
|
||||
// job was enqueued. Always present for jobs enqueued post-deploy.
|
||||
// 2. instance.lastScheduledTimestamp — backward-compat fallback for
|
||||
// in-flight Redis jobs enqueued by older engines that didn't
|
||||
// include lastScheduleTime in the payload. Once those drain
|
||||
// this fallback never triggers and we can drop the column.
|
||||
// 3. undefined — first-ever fire (no previous fire to point at).
|
||||
const lastTimestamp =
|
||||
params.lastScheduleTime ?? instance.lastScheduledTimestamp ?? undefined;
|
||||
|
||||
const payload = {
|
||||
scheduleId: instance.taskSchedule.friendlyId,
|
||||
type: instance.taskSchedule.type as "DECLARATIVE" | "IMPERATIVE",
|
||||
timestamp: scheduleTimestamp,
|
||||
lastTimestamp,
|
||||
externalId: instance.taskSchedule.externalId ?? undefined,
|
||||
timezone: instance.taskSchedule.timezone,
|
||||
upcoming: nextScheduledTimestamps(
|
||||
instance.taskSchedule.generatorExpression,
|
||||
instance.taskSchedule.timezone,
|
||||
scheduleTimestamp,
|
||||
10
|
||||
),
|
||||
};
|
||||
|
||||
// Calculate execution timing metrics
|
||||
const actualExecutionTime = new Date();
|
||||
const schedulingAccuracyMs = actualExecutionTime.getTime() - scheduleTimestamp.getTime();
|
||||
|
||||
span.setAttribute("scheduling_accuracy_ms", schedulingAccuracyMs);
|
||||
span.setAttribute("actual_execution_time", actualExecutionTime.toISOString());
|
||||
|
||||
this.logger.debug("Triggering scheduled task", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
scheduleTimestamp: scheduleTimestamp.toISOString(),
|
||||
actualExecutionTime: actualExecutionTime.toISOString(),
|
||||
schedulingAccuracyMs,
|
||||
lastTimestamp: lastTimestamp?.toISOString(),
|
||||
});
|
||||
|
||||
const triggerStartTime = Date.now();
|
||||
|
||||
// Rewritten try/catch to use tryCatch utility
|
||||
const [triggerError, result] = await tryCatch(
|
||||
this.onTriggerScheduledTask({
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
environment: instance.environment,
|
||||
payload,
|
||||
scheduleInstanceId: instance.id,
|
||||
scheduleId: instance.taskSchedule.id,
|
||||
exactScheduleTime: scheduleTimestamp,
|
||||
})
|
||||
);
|
||||
|
||||
const triggerDuration = Date.now() - triggerStartTime;
|
||||
|
||||
this.scheduleExecutionDuration.record(triggerDuration, {
|
||||
environment_type: environmentType,
|
||||
schedule_type: scheduleType,
|
||||
});
|
||||
|
||||
if (triggerError) {
|
||||
this.logger.error("Error calling trigger callback", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
durationMs: triggerDuration,
|
||||
error: triggerError instanceof Error ? triggerError.message : String(triggerError),
|
||||
});
|
||||
|
||||
this.scheduleExecutionFailureCounter.add(1, {
|
||||
environment_type: environmentType,
|
||||
schedule_type: scheduleType,
|
||||
error_type: "callback_error",
|
||||
});
|
||||
|
||||
span.setAttribute("trigger_error", true);
|
||||
span.setAttribute(
|
||||
"trigger_error_message",
|
||||
triggerError instanceof Error ? triggerError.message : String(triggerError)
|
||||
);
|
||||
} else if (result) {
|
||||
if (result.success) {
|
||||
this.logger.debug("Successfully triggered scheduled task", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
durationMs: triggerDuration,
|
||||
});
|
||||
|
||||
this.scheduleExecutionCounter.add(1, {
|
||||
environment_type: environmentType,
|
||||
schedule_type: scheduleType,
|
||||
status: "success",
|
||||
});
|
||||
|
||||
span.setAttribute("trigger_success", true);
|
||||
} else {
|
||||
// QUEUE_LIMIT and OUT_OF_ENTITLEMENTS are expected,
|
||||
// non-actionable outcomes (the environment is at its queue limit,
|
||||
// or the org is out of entitlements). Log them as warnings so they
|
||||
// aren't reported as errors, while still recording the metric.
|
||||
const isExpectedFailure =
|
||||
result.errorType === "QUEUE_LIMIT" || result.errorType === "OUT_OF_ENTITLEMENTS";
|
||||
|
||||
if (isExpectedFailure) {
|
||||
this.logger.warn("Scheduled task trigger skipped", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
durationMs: triggerDuration,
|
||||
errorType: result.errorType,
|
||||
error: result.error,
|
||||
});
|
||||
} else {
|
||||
this.logger.error("Failed to trigger scheduled task", {
|
||||
instanceId: params.instanceId,
|
||||
taskIdentifier: instance.taskSchedule.taskIdentifier,
|
||||
durationMs: triggerDuration,
|
||||
error: result.error,
|
||||
});
|
||||
}
|
||||
|
||||
const failureErrorType =
|
||||
result.errorType === "QUEUE_LIMIT"
|
||||
? "queue_limit"
|
||||
: result.errorType === "OUT_OF_ENTITLEMENTS"
|
||||
? "out_of_entitlements"
|
||||
: "task_failure";
|
||||
|
||||
this.scheduleExecutionFailureCounter.add(1, {
|
||||
environment_type: environmentType,
|
||||
schedule_type: scheduleType,
|
||||
error_type: failureErrorType,
|
||||
});
|
||||
|
||||
span.setAttribute("trigger_success", false);
|
||||
if (result.error) {
|
||||
span.setAttribute("trigger_error_message", result.error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
span.setAttribute("trigger_duration_ms", triggerDuration);
|
||||
} else {
|
||||
this.logger.debug("Skipping task trigger due to conditions", {
|
||||
instanceId: params.instanceId,
|
||||
reason: skipReason,
|
||||
});
|
||||
|
||||
this.scheduleExecutionCounter.add(1, {
|
||||
environment_type: environmentType ?? "unknown",
|
||||
schedule_type: scheduleType ?? "unknown",
|
||||
status: "skipped",
|
||||
});
|
||||
}
|
||||
|
||||
// Register the next run. `fromTimestamp` advances on every tick so
|
||||
// the next cron slot keeps marching forward even through skips.
|
||||
// `lastScheduleTime` is the actual previous fire time the next job
|
||||
// will report as `payload.lastTimestamp` — only advance it when we
|
||||
// actually triggered, otherwise carry forward the existing value so
|
||||
// a long pause/disconnect doesn't quietly overwrite the real
|
||||
// last-fire timestamp with a series of skipped slots.
|
||||
const carriedLastScheduleTime = shouldTrigger
|
||||
? scheduleTimestamp
|
||||
: (params.lastScheduleTime ?? instance.lastScheduledTimestamp ?? undefined);
|
||||
|
||||
const [nextRunError] = await tryCatch(
|
||||
this.registerNextTaskScheduleInstance({
|
||||
instanceId: params.instanceId,
|
||||
fromTimestamp: scheduleTimestamp,
|
||||
lastScheduleTime: carriedLastScheduleTime,
|
||||
})
|
||||
);
|
||||
if (nextRunError) {
|
||||
this.logger.error("Failed to schedule next run after execution", {
|
||||
instanceId: params.instanceId,
|
||||
error: nextRunError instanceof Error ? nextRunError.message : String(nextRunError),
|
||||
});
|
||||
|
||||
span.setAttribute("next_run_registration_error", true);
|
||||
span.setAttribute(
|
||||
"next_run_error_message",
|
||||
nextRunError instanceof Error ? nextRunError.message : String(nextRunError)
|
||||
);
|
||||
|
||||
if (!params.finalAttempt) {
|
||||
throw nextRunError;
|
||||
}
|
||||
} else {
|
||||
span.setAttribute("next_run_registered", true);
|
||||
}
|
||||
|
||||
const totalDuration = Date.now() - startTime;
|
||||
this.logger.debug("Scheduled task trigger completed", {
|
||||
instanceId: params.instanceId,
|
||||
totalDurationMs: totalDuration,
|
||||
});
|
||||
|
||||
span.setAttribute("total_duration_ms", totalDuration);
|
||||
span.setAttribute("success", true);
|
||||
} catch (error) {
|
||||
const totalDuration = Date.now() - startTime;
|
||||
this.logger.error("Failed to trigger scheduled task", {
|
||||
instanceId: params.instanceId,
|
||||
totalDurationMs: totalDuration,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
|
||||
this.scheduleExecutionFailureCounter.add(1, {
|
||||
environment_type: environmentType ?? "unknown",
|
||||
schedule_type: scheduleType ?? "unknown",
|
||||
error_type: "system_error",
|
||||
});
|
||||
|
||||
span.setAttribute("error", true);
|
||||
span.setAttribute("error_message", error instanceof Error ? error.message : String(error));
|
||||
span.setAttribute("total_duration_ms", totalDuration);
|
||||
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueues a scheduled task with distributed execution timing
|
||||
*/
|
||||
private async enqueueScheduledTask(
|
||||
instanceId: string,
|
||||
exactScheduleTime: Date,
|
||||
lastScheduleTime?: Date
|
||||
) {
|
||||
return startSpan(this.tracer, "enqueueScheduledTask", async (span) => {
|
||||
span.setAttribute("instanceId", instanceId);
|
||||
span.setAttribute("exactScheduleTime", exactScheduleTime.toISOString());
|
||||
if (lastScheduleTime) {
|
||||
span.setAttribute("lastScheduleTime", lastScheduleTime.toISOString());
|
||||
}
|
||||
|
||||
const distributedExecutionTime = calculateDistributedExecutionTime(
|
||||
exactScheduleTime,
|
||||
this.distributionWindowSeconds,
|
||||
instanceId
|
||||
);
|
||||
|
||||
const distributionOffsetMs = exactScheduleTime.getTime() - distributedExecutionTime.getTime();
|
||||
|
||||
span.setAttribute("distributedExecutionTime", distributedExecutionTime.toISOString());
|
||||
span.setAttribute("distributionOffsetMs", distributionOffsetMs);
|
||||
span.setAttribute("distributionWindowSeconds", this.distributionWindowSeconds);
|
||||
|
||||
this.distributionOffsetHistogram.record(distributionOffsetMs, {
|
||||
distribution_window_seconds: this.distributionWindowSeconds.toString(),
|
||||
});
|
||||
|
||||
this.logger.debug("Enqueuing scheduled task with distributed execution", {
|
||||
instanceId,
|
||||
exactScheduleTime: exactScheduleTime.toISOString(),
|
||||
distributedExecutionTime: distributedExecutionTime.toISOString(),
|
||||
distributionOffsetMs,
|
||||
distributionWindowSeconds: this.distributionWindowSeconds,
|
||||
});
|
||||
|
||||
try {
|
||||
await this.worker.enqueue({
|
||||
id: `scheduled-task-instance:${instanceId}`,
|
||||
job: "schedule.triggerScheduledTask",
|
||||
payload: {
|
||||
instanceId,
|
||||
exactScheduleTime,
|
||||
lastScheduleTime,
|
||||
},
|
||||
availableAt: distributedExecutionTime,
|
||||
});
|
||||
|
||||
span.setAttribute("enqueue_success", true);
|
||||
|
||||
this.logger.debug("Successfully enqueued scheduled task", {
|
||||
instanceId,
|
||||
jobId: `scheduled-task-instance:${instanceId}`,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error("Failed to enqueue scheduled task", {
|
||||
instanceId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
|
||||
span.setAttribute("enqueue_error", true);
|
||||
span.setAttribute(
|
||||
"enqueue_error_message",
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public recoverSchedulesInEnvironment(projectId: string, environmentId: string) {
|
||||
return startSpan(this.tracer, "recoverSchedulesInEnvironment", async (span) => {
|
||||
this.logger.info("Recovering schedules in environment", {
|
||||
environmentId,
|
||||
projectId,
|
||||
});
|
||||
|
||||
span.setAttribute("environmentId", environmentId);
|
||||
|
||||
const schedules = await this.prisma.taskSchedule.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
instances: {
|
||||
some: {
|
||||
environmentId,
|
||||
},
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
generatorExpression: true,
|
||||
timezone: true,
|
||||
instances: {
|
||||
select: {
|
||||
id: true,
|
||||
environmentId: true,
|
||||
createdAt: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const instancesWithSchedule = schedules
|
||||
.map((schedule) => ({
|
||||
schedule,
|
||||
instance: schedule.instances.find((instance) => instance.environmentId === environmentId),
|
||||
}))
|
||||
.filter((instance) => instance.instance) as Array<{
|
||||
schedule: Omit<(typeof schedules)[number], "instances">;
|
||||
instance: NonNullable<(typeof schedules)[number]["instances"][number]>;
|
||||
}>;
|
||||
|
||||
if (instancesWithSchedule.length === 0) {
|
||||
this.logger.info("No instances found for environment", {
|
||||
environmentId,
|
||||
projectId,
|
||||
});
|
||||
|
||||
return {
|
||||
recovered: [],
|
||||
skipped: [],
|
||||
};
|
||||
}
|
||||
|
||||
const results = {
|
||||
recovered: [],
|
||||
skipped: [],
|
||||
} as { recovered: string[]; skipped: string[] };
|
||||
|
||||
for (const { instance, schedule } of instancesWithSchedule) {
|
||||
this.logger.debug("Recovering schedule", {
|
||||
schedule,
|
||||
instance,
|
||||
});
|
||||
|
||||
const [recoverError, result] = await tryCatch(
|
||||
this.#recoverTaskScheduleInstance({ instance, schedule })
|
||||
);
|
||||
|
||||
if (recoverError) {
|
||||
this.logger.error("Error recovering schedule", {
|
||||
error: recoverError instanceof Error ? recoverError.message : String(recoverError),
|
||||
});
|
||||
|
||||
span.setAttribute("recover_error", true);
|
||||
span.setAttribute(
|
||||
"recover_error_message",
|
||||
recoverError instanceof Error ? recoverError.message : String(recoverError)
|
||||
);
|
||||
} else {
|
||||
span.setAttribute("recover_success", true);
|
||||
|
||||
if (result === "recovered") {
|
||||
results.recovered.push(instance.id);
|
||||
} else {
|
||||
results.skipped.push(instance.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
});
|
||||
}
|
||||
|
||||
async #recoverTaskScheduleInstance({
|
||||
instance,
|
||||
schedule,
|
||||
}: {
|
||||
instance: {
|
||||
id: string;
|
||||
environmentId: string;
|
||||
createdAt: Date;
|
||||
};
|
||||
schedule: { id: string; generatorExpression: string; timezone: string | null };
|
||||
}) {
|
||||
// inspect the schedule worker to see if there is a job for this instance
|
||||
const job = await this.worker.getJob(`scheduled-task-instance:${instance.id}`);
|
||||
|
||||
if (job) {
|
||||
this.logger.debug("Job already exists for instance", {
|
||||
instanceId: instance.id,
|
||||
job,
|
||||
schedule,
|
||||
});
|
||||
|
||||
return "skipped";
|
||||
}
|
||||
|
||||
this.logger.debug("No job found for instance, registering next run", {
|
||||
instanceId: instance.id,
|
||||
schedule,
|
||||
});
|
||||
|
||||
// No `lastScheduleTime` passed — `registerNextTaskScheduleInstance`
|
||||
// will derive it from the cron's previous slot (with a createdAt
|
||||
// guard) so the post-recovery fire reports an accurate
|
||||
// `payload.lastTimestamp`.
|
||||
await this.registerNextTaskScheduleInstance({ instanceId: instance.id });
|
||||
|
||||
return "recovered";
|
||||
}
|
||||
|
||||
async getJob(id: string) {
|
||||
return this.worker.getJob(id);
|
||||
}
|
||||
|
||||
async quit() {
|
||||
this.logger.info("Shutting down schedule engine");
|
||||
|
||||
try {
|
||||
await this.worker.stop();
|
||||
this.logger.info("Schedule engine worker stopped successfully");
|
||||
} catch (error) {
|
||||
this.logger.error("Error stopping schedule engine worker", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { parseExpression } from "cron-parser";
|
||||
|
||||
export function calculateNextScheduledTimestampFromNow(schedule: string, timezone: string | null) {
|
||||
return calculateNextScheduledTimestamp(schedule, timezone, new Date());
|
||||
}
|
||||
|
||||
export function calculateNextScheduledTimestamp(
|
||||
schedule: string,
|
||||
timezone: string | null,
|
||||
lastScheduledTimestamp: Date = new Date()
|
||||
) {
|
||||
const nextStep = calculateNextStep(schedule, timezone, lastScheduledTimestamp);
|
||||
|
||||
if (nextStep.getTime() < Date.now()) {
|
||||
// If the next step is in the past, we just need to calculate the next step from now
|
||||
return calculateNextStep(schedule, timezone, new Date());
|
||||
}
|
||||
|
||||
return nextStep;
|
||||
}
|
||||
|
||||
function calculateNextStep(schedule: string, timezone: string | null, currentDate: Date) {
|
||||
return parseExpression(schedule, {
|
||||
currentDate,
|
||||
utc: timezone === null,
|
||||
tz: timezone ?? undefined,
|
||||
})
|
||||
.next()
|
||||
.toDate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cron's previous slot relative to `fromTimestamp`. For a continuously-
|
||||
* running schedule this equals the actual last fire time; for paused or
|
||||
* DST-edge cases it's an approximation. Used only on the recovery path
|
||||
* where the actual last fire isn't recoverable from in-flight worker state.
|
||||
*/
|
||||
export function previousScheduledTimestamp(
|
||||
cron: string,
|
||||
timezone: string | null,
|
||||
fromTimestamp: Date = new Date()
|
||||
) {
|
||||
return parseExpression(cron, {
|
||||
currentDate: fromTimestamp,
|
||||
utc: timezone === null,
|
||||
tz: timezone ?? undefined,
|
||||
})
|
||||
.prev()
|
||||
.toDate();
|
||||
}
|
||||
|
||||
export function nextScheduledTimestamps(
|
||||
cron: string,
|
||||
timezone: string | null,
|
||||
lastScheduledTimestamp: Date,
|
||||
count: number = 1
|
||||
) {
|
||||
const result: Array<Date> = [];
|
||||
let nextScheduledTimestamp = lastScheduledTimestamp;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
nextScheduledTimestamp = calculateNextScheduledTimestamp(
|
||||
cron,
|
||||
timezone,
|
||||
nextScheduledTimestamp
|
||||
);
|
||||
|
||||
result.push(nextScheduledTimestamp);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { Logger } from "@trigger.dev/core/logger";
|
||||
import type { Meter, Tracer } from "@internal/tracing";
|
||||
import type { Prisma, PrismaClient } from "@trigger.dev/database";
|
||||
import type { RedisOptions } from "@internal/redis";
|
||||
|
||||
export type SchedulingEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
|
||||
include: { project: true; organization: true; orgMember: true };
|
||||
}>;
|
||||
|
||||
export type TriggerScheduledTaskParams = {
|
||||
taskIdentifier: string;
|
||||
environment: SchedulingEnvironment;
|
||||
payload: {
|
||||
scheduleId: string;
|
||||
type: "DECLARATIVE" | "IMPERATIVE";
|
||||
timestamp: Date;
|
||||
lastTimestamp?: Date;
|
||||
externalId?: string;
|
||||
timezone: string;
|
||||
upcoming: Date[];
|
||||
};
|
||||
scheduleInstanceId: string;
|
||||
scheduleId: string;
|
||||
exactScheduleTime?: Date;
|
||||
};
|
||||
|
||||
export type TriggerScheduledTaskErrorType = "QUEUE_LIMIT" | "OUT_OF_ENTITLEMENTS" | "SYSTEM_ERROR";
|
||||
|
||||
export interface TriggerScheduledTaskCallback {
|
||||
(params: TriggerScheduledTaskParams): Promise<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
errorType?: TriggerScheduledTaskErrorType;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ScheduleEngineOptions {
|
||||
logger?: Logger;
|
||||
logLevel?: string;
|
||||
prisma: PrismaClient;
|
||||
redis: RedisOptions;
|
||||
worker: {
|
||||
concurrency: number;
|
||||
workers?: number;
|
||||
tasksPerWorker?: number;
|
||||
pollIntervalMs?: number;
|
||||
shutdownTimeoutMs?: number;
|
||||
disabled?: boolean;
|
||||
};
|
||||
distributionWindow?: {
|
||||
seconds: number;
|
||||
};
|
||||
tracer?: Tracer;
|
||||
meter?: Meter;
|
||||
onTriggerScheduledTask: TriggerScheduledTaskCallback;
|
||||
isDevEnvironmentConnectedHandler: (environmentId: string) => Promise<boolean>;
|
||||
onRegisterScheduleInstance?: (instanceId: string) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface UpsertScheduleParams {
|
||||
projectId: string;
|
||||
schedule: {
|
||||
friendlyId?: string;
|
||||
taskIdentifier: string;
|
||||
deduplicationKey?: string;
|
||||
cron: string;
|
||||
timezone?: string;
|
||||
externalId?: string;
|
||||
environments: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface TriggerScheduleParams {
|
||||
instanceId: string;
|
||||
finalAttempt: boolean;
|
||||
exactScheduleTime?: Date;
|
||||
lastScheduleTime?: Date;
|
||||
}
|
||||
|
||||
export interface RegisterScheduleInstanceParams {
|
||||
instanceId: string;
|
||||
/**
|
||||
* Anchor for computing the next cron slot. Defaults to now() when omitted.
|
||||
* This advances on every tick (fired or skipped) so the next slot keeps
|
||||
* marching forward regardless of skip reasons.
|
||||
*/
|
||||
fromTimestamp?: Date;
|
||||
/**
|
||||
* The actual previous fire time to embed in the next worker job's payload,
|
||||
* which becomes that job's `payload.lastTimestamp` on dequeue. Distinct
|
||||
* from `fromTimestamp` so that skipped ticks (inactive schedule, dev env
|
||||
* disconnected, etc.) do NOT advance this — only real fires do.
|
||||
*/
|
||||
lastScheduleTime?: Date;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const scheduleWorkerCatalog = {
|
||||
"schedule.triggerScheduledTask": {
|
||||
schema: z.object({
|
||||
instanceId: z.string(),
|
||||
exactScheduleTime: z.coerce.date(),
|
||||
// Optional for backward compat with in-flight jobs enqueued by older
|
||||
// engines. After deploy, every newly-enqueued job populates this with
|
||||
// the just-fired schedule time so the next dequeue can report
|
||||
// payload.lastTimestamp accurately without a DB round-trip.
|
||||
lastScheduleTime: z.coerce.date().optional(),
|
||||
}),
|
||||
visibilityTimeoutMs: 60_000,
|
||||
retry: {
|
||||
maxAttempts: 5,
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,7 @@
|
||||
export { ScheduleEngine } from "./engine/index.js";
|
||||
export type {
|
||||
ScheduleEngineOptions,
|
||||
TriggerScheduleParams,
|
||||
TriggerScheduledTaskCallback,
|
||||
TriggerScheduledTaskErrorType,
|
||||
} from "./engine/types.js";
|
||||
@@ -0,0 +1,242 @@
|
||||
import { containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@internal/tracing";
|
||||
import { setTimeout } from "timers/promises";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
|
||||
import { ScheduleEngine } from "../src/index.js";
|
||||
|
||||
describe("ScheduleEngine Integration", () => {
|
||||
containerTest(
|
||||
"should process full schedule lifecycle through worker with multiple executions",
|
||||
{ timeout: 240_000 }, // Increase timeout for multiple executions (4 minutes)
|
||||
async ({ prisma, redisOptions }) => {
|
||||
// Real callback function for testing expectations
|
||||
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
|
||||
|
||||
const triggerCalls: Array<{
|
||||
params: TriggerScheduledTaskParams;
|
||||
executionTime: Date;
|
||||
}> = [];
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: false, // Enable worker for full integration test
|
||||
pollIntervalMs: 100, // Poll frequently for faster test execution
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
const executionTime = new Date(); // Capture when callback is actually called
|
||||
console.log(
|
||||
`TriggerScheduledTask called at: ${executionTime.toISOString()} (execution #${
|
||||
triggerCalls.length + 1
|
||||
})`
|
||||
);
|
||||
console.log("TriggerScheduledTask", params);
|
||||
|
||||
triggerCalls.push({ params, executionTime });
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
|
||||
});
|
||||
|
||||
try {
|
||||
// Create real database records
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Test Organization",
|
||||
slug: "test-org",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Test Project",
|
||||
slug: "test-project",
|
||||
externalRef: "test-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "test-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_test_1234",
|
||||
pkApiKey: "pk_test_1234",
|
||||
shortcode: "test-short",
|
||||
},
|
||||
});
|
||||
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_abc123",
|
||||
taskIdentifier: "test-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "test-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: "* * * * *", // Every minute
|
||||
generatorDescription: "Every minute",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
externalId: "ext-123",
|
||||
},
|
||||
});
|
||||
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Manually enqueue the first scheduled task to kick off the lifecycle.
|
||||
// Anchor expectations to the first observed `exactScheduleTime` rather
|
||||
// than a precomputed wall-clock value — registration that happens to
|
||||
// straddle a minute boundary used to flake tests asserting against a
|
||||
// pre-baked "next minute".
|
||||
await engine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
|
||||
|
||||
// Wait for the first execution
|
||||
console.log("Waiting for first execution...");
|
||||
const startTime = Date.now();
|
||||
const maxWaitTime = 70_000; // 70 seconds max wait for first execution
|
||||
|
||||
while (triggerCalls.length === 0 && Date.now() - startTime < maxWaitTime) {
|
||||
await setTimeout(100);
|
||||
}
|
||||
|
||||
expect(triggerCalls.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// Verify the first execution
|
||||
const firstExecution = triggerCalls[0];
|
||||
console.log("First execution verified, waiting for second execution...");
|
||||
|
||||
// Wait for the second execution (should happen ~1 minute after the first)
|
||||
const secondExecutionStartTime = Date.now();
|
||||
const maxWaitForSecond = 80_000; // 80 seconds max wait for second execution
|
||||
|
||||
while (
|
||||
triggerCalls.length < 2 &&
|
||||
Date.now() - secondExecutionStartTime < maxWaitForSecond
|
||||
) {
|
||||
await setTimeout(100);
|
||||
}
|
||||
|
||||
expect(triggerCalls.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const secondExecution = triggerCalls[1];
|
||||
console.log("Second execution verified!");
|
||||
|
||||
// Give a small delay for database updates to complete before checking
|
||||
await setTimeout(500);
|
||||
|
||||
// Verify both executions have correct timing and distribution window behavior
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const execution = triggerCalls[i];
|
||||
const expectedScheduleTime = execution.params.exactScheduleTime;
|
||||
|
||||
if (expectedScheduleTime) {
|
||||
// Calculate the distribution window (10 seconds before the scheduled time)
|
||||
const distributionWindowStart = new Date(expectedScheduleTime);
|
||||
distributionWindowStart.setSeconds(distributionWindowStart.getSeconds() - 10);
|
||||
|
||||
console.log(`Execution ${i + 1}:`);
|
||||
console.log(" Scheduled time:", expectedScheduleTime.toISOString());
|
||||
console.log(" Distribution window start:", distributionWindowStart.toISOString());
|
||||
console.log(" Actual execution time:", execution.executionTime.toISOString());
|
||||
|
||||
// Verify the callback was executed within the distribution window
|
||||
expect(execution.executionTime.getTime()).toBeGreaterThanOrEqual(
|
||||
distributionWindowStart.getTime()
|
||||
);
|
||||
expect(execution.executionTime.getTime()).toBeLessThanOrEqual(
|
||||
expectedScheduleTime.getTime()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Anchor all expectations to what the engine actually fired with, so
|
||||
// the test stays deterministic regardless of when within a minute it
|
||||
// started.
|
||||
const firstScheduledTime = firstExecution.params.exactScheduleTime;
|
||||
const secondScheduledTime = secondExecution.params.exactScheduleTime;
|
||||
expect(firstScheduledTime).toBeDefined();
|
||||
expect(secondScheduledTime).toBeDefined();
|
||||
|
||||
// Each cron slot for "* * * * *" is exactly 60s apart.
|
||||
expect(secondScheduledTime!.getTime() - firstScheduledTime!.getTime()).toBe(60_000);
|
||||
|
||||
// Verify the first execution parameters
|
||||
expect(firstExecution.params).toEqual({
|
||||
taskIdentifier: "test-task",
|
||||
environment: expect.objectContaining({
|
||||
id: environment.id,
|
||||
type: "PRODUCTION",
|
||||
project: expect.objectContaining({
|
||||
id: project.id,
|
||||
name: "Test Project",
|
||||
slug: "test-project",
|
||||
}),
|
||||
organization: expect.objectContaining({
|
||||
id: organization.id,
|
||||
title: "Test Organization",
|
||||
slug: "test-org",
|
||||
}),
|
||||
}),
|
||||
payload: {
|
||||
scheduleId: "sched_abc123",
|
||||
type: "DECLARATIVE",
|
||||
timestamp: firstScheduledTime,
|
||||
// First-ever fire: no `lastScheduleTime` carried in the worker
|
||||
// payload and `instance.lastScheduledTimestamp` is null on a
|
||||
// fresh instance, so lastTimestamp is undefined. This preserves
|
||||
// the `if (!payload.lastTimestamp)` first-run sentinel customers
|
||||
// rely on.
|
||||
lastTimestamp: undefined,
|
||||
externalId: "ext-123",
|
||||
timezone: "UTC",
|
||||
upcoming: expect.arrayContaining([expect.any(Date)]),
|
||||
},
|
||||
scheduleInstanceId: scheduleInstance.id,
|
||||
scheduleId: taskSchedule.id,
|
||||
exactScheduleTime: firstScheduledTime,
|
||||
});
|
||||
|
||||
// Verify the second execution parameters
|
||||
expect(secondExecution.params).toEqual({
|
||||
taskIdentifier: "test-task",
|
||||
environment: expect.objectContaining({
|
||||
id: environment.id,
|
||||
type: "PRODUCTION",
|
||||
}),
|
||||
payload: {
|
||||
scheduleId: "sched_abc123",
|
||||
type: "DECLARATIVE",
|
||||
timestamp: secondScheduledTime,
|
||||
// The previous fire's exactScheduleTime is carried through the
|
||||
// worker payload as `lastScheduleTime` and surfaced here.
|
||||
lastTimestamp: firstScheduledTime,
|
||||
externalId: "ext-123",
|
||||
timezone: "UTC",
|
||||
upcoming: expect.arrayContaining([expect.any(Date)]),
|
||||
},
|
||||
scheduleInstanceId: scheduleInstance.id,
|
||||
scheduleId: taskSchedule.id,
|
||||
exactScheduleTime: secondScheduledTime,
|
||||
});
|
||||
} finally {
|
||||
// Clean up: stop the worker
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import { containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@internal/tracing";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
|
||||
import { ScheduleEngine } from "../src/index.js";
|
||||
|
||||
describe("ScheduleEngine Integration (part 2)", () => {
|
||||
// Deploy-moment backward compatibility. At deploy time, in-flight Redis jobs
|
||||
// were enqueued by the old engine — their payload has no `lastScheduleTime`
|
||||
// field — and `instance.lastScheduledTimestamp` is still populated (last
|
||||
// written by the old engine pre-deploy). The new engine must report that DB
|
||||
// value as `payload.lastTimestamp` so customers don't see a transient
|
||||
// `undefined` for the one fire per schedule that drains the legacy queue.
|
||||
containerTest(
|
||||
"should fall back to instance.lastScheduledTimestamp when payload lacks lastScheduleTime",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const triggerCalls: TriggerScheduledTaskParams[] = [];
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: true, // Don't actually run the worker — calling triggerScheduledTask directly
|
||||
pollIntervalMs: 1000,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
triggerCalls.push(params);
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
try {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: "Legacy Payload Org", slug: "legacy-payload-org" },
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Legacy Payload Project",
|
||||
slug: "legacy-payload-project",
|
||||
externalRef: "legacy-payload-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "legacy-payload-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_legacy_1234",
|
||||
pkApiKey: "pk_legacy_1234",
|
||||
shortcode: "legacy-short",
|
||||
},
|
||||
});
|
||||
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_legacy_payload",
|
||||
taskIdentifier: "legacy-payload-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "legacy-payload-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: "*/5 * * * *",
|
||||
generatorDescription: "Every 5 minutes",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
externalId: "legacy-ext",
|
||||
},
|
||||
});
|
||||
|
||||
// Pre-populate lastScheduledTimestamp on the instance — simulates the
|
||||
// value the old engine wrote to the DB before this PR deployed.
|
||||
const preDeployLastFire = new Date("2026-04-30T10:00:00.000Z");
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
lastScheduledTimestamp: preDeployLastFire,
|
||||
},
|
||||
});
|
||||
|
||||
// Call triggerScheduledTask directly without lastScheduleTime,
|
||||
// simulating an in-flight Redis job enqueued by the old engine.
|
||||
const exactScheduleTime = new Date("2026-04-30T10:05:00.000Z");
|
||||
await engine.triggerScheduledTask({
|
||||
instanceId: scheduleInstance.id,
|
||||
finalAttempt: false,
|
||||
exactScheduleTime,
|
||||
// lastScheduleTime intentionally omitted — legacy payload shape
|
||||
});
|
||||
|
||||
expect(triggerCalls.length).toBe(1);
|
||||
expect(triggerCalls[0].payload.timestamp).toEqual(exactScheduleTime);
|
||||
// Falls back to instance.lastScheduledTimestamp from the DB rather
|
||||
// than reporting undefined for this one transitional fire.
|
||||
expect(triggerCalls[0].payload.lastTimestamp).toEqual(preDeployLastFire);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,580 @@
|
||||
import { containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@internal/tracing";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import type { TriggerScheduledTaskParams } from "../src/engine/types.js";
|
||||
import { ScheduleEngine } from "../src/index.js";
|
||||
|
||||
describe("Schedule Recovery", () => {
|
||||
containerTest(
|
||||
"should recover schedules when no existing jobs are found",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
|
||||
const triggerCalls: TriggerScheduledTaskParams[] = [];
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: true, // Disable worker to prevent automatic execution
|
||||
pollIntervalMs: 1000,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
triggerCalls.push(params);
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
|
||||
});
|
||||
|
||||
try {
|
||||
// Create test data
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Recovery Test Org",
|
||||
slug: "recovery-test-org",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Recovery Test Project",
|
||||
slug: "recovery-test-project",
|
||||
externalRef: "recovery-test-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "recovery-test-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_recovery_test_1234",
|
||||
pkApiKey: "pk_recovery_test_1234",
|
||||
shortcode: "recovery-test-short",
|
||||
},
|
||||
});
|
||||
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_recovery_123",
|
||||
taskIdentifier: "recovery-test-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "recovery-test-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: "0 */5 * * *", // Every 5 minutes
|
||||
generatorDescription: "Every 5 minutes",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
externalId: "recovery-ext-123",
|
||||
},
|
||||
});
|
||||
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Verify no job exists initially
|
||||
const jobBeforeRecovery = await engine.getJob(
|
||||
`scheduled-task-instance:${scheduleInstance.id}`
|
||||
);
|
||||
expect(jobBeforeRecovery).toBeNull();
|
||||
|
||||
// Perform recovery
|
||||
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
|
||||
|
||||
// Verify that a job was created. The engine no longer persists
|
||||
// nextScheduledTimestamp; correctness is now determined entirely by the
|
||||
// job sitting in the worker queue.
|
||||
const jobAfterRecovery = await engine.getJob(
|
||||
`scheduled-task-instance:${scheduleInstance.id}`
|
||||
);
|
||||
expect(jobAfterRecovery).not.toBeNull();
|
||||
expect(jobAfterRecovery?.job).toBe("schedule.triggerScheduledTask");
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"should not create duplicate jobs when schedule already has an active job",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
|
||||
const triggerCalls: TriggerScheduledTaskParams[] = [];
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: true, // Disable worker to prevent automatic execution
|
||||
pollIntervalMs: 1000,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
triggerCalls.push(params);
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
|
||||
});
|
||||
|
||||
try {
|
||||
// Create test data
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Duplicate Test Org",
|
||||
slug: "duplicate-test-org",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Duplicate Test Project",
|
||||
slug: "duplicate-test-project",
|
||||
externalRef: "duplicate-test-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "duplicate-test-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_duplicate_test_1234",
|
||||
pkApiKey: "pk_duplicate_test_1234",
|
||||
shortcode: "duplicate-test-short",
|
||||
},
|
||||
});
|
||||
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_duplicate_123",
|
||||
taskIdentifier: "duplicate-test-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "duplicate-test-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: "0 */10 * * *", // Every 10 minutes
|
||||
generatorDescription: "Every 10 minutes",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
externalId: "duplicate-ext-123",
|
||||
},
|
||||
});
|
||||
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
// First, register the schedule normally
|
||||
await engine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
|
||||
|
||||
// Verify job exists
|
||||
const jobAfterFirstRegistration = await engine.getJob(
|
||||
`scheduled-task-instance:${scheduleInstance.id}`
|
||||
);
|
||||
expect(jobAfterFirstRegistration).not.toBeNull();
|
||||
const firstJobId = jobAfterFirstRegistration?.id;
|
||||
|
||||
// Now run recovery - it should not create a duplicate job
|
||||
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
|
||||
|
||||
// Verify the same job still exists (no duplicate created)
|
||||
const jobAfterRecovery = await engine.getJob(
|
||||
`scheduled-task-instance:${scheduleInstance.id}`
|
||||
);
|
||||
expect(jobAfterRecovery).not.toBeNull();
|
||||
expect(jobAfterRecovery?.id).toBe(firstJobId);
|
||||
expect(jobAfterRecovery?.deduplicationKey).toBe(jobAfterFirstRegistration.deduplicationKey);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"should recover multiple schedules in the same environment",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
|
||||
const triggerCalls: TriggerScheduledTaskParams[] = [];
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: true, // Disable worker to prevent automatic execution
|
||||
pollIntervalMs: 1000,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
triggerCalls.push(params);
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
|
||||
});
|
||||
|
||||
try {
|
||||
// Create test data
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Multiple Test Org",
|
||||
slug: "multiple-test-org",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Multiple Test Project",
|
||||
slug: "multiple-test-project",
|
||||
externalRef: "multiple-test-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "multiple-test-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_multiple_test_1234",
|
||||
pkApiKey: "pk_multiple_test_1234",
|
||||
shortcode: "multiple-test-short",
|
||||
},
|
||||
});
|
||||
|
||||
// Create multiple task schedules
|
||||
const schedules = [];
|
||||
const instances = [];
|
||||
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: `sched_multiple_${i}`,
|
||||
taskIdentifier: `multiple-test-task-${i}`,
|
||||
projectId: project.id,
|
||||
deduplicationKey: `multiple-test-dedup-${i}`,
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: `${i} */15 * * *`, // Every 15 minutes at different minute offsets
|
||||
generatorDescription: `Every 15 minutes (${i})`,
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
externalId: `multiple-ext-${i}`,
|
||||
},
|
||||
});
|
||||
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
schedules.push(taskSchedule);
|
||||
instances.push(scheduleInstance);
|
||||
}
|
||||
|
||||
// Verify no jobs exist initially
|
||||
for (const instance of instances) {
|
||||
const job = await engine.getJob(`scheduled-task-instance:${instance.id}`);
|
||||
expect(job).toBeNull();
|
||||
}
|
||||
|
||||
// Perform recovery
|
||||
await engine.recoverSchedulesInEnvironment(project.id, environment.id);
|
||||
|
||||
// Verify that jobs were created for all instances. The engine no longer
|
||||
// persists nextScheduledTimestamp — the worker-queue presence is the
|
||||
// source of truth.
|
||||
for (const instance of instances) {
|
||||
const job = await engine.getJob(`scheduled-task-instance:${instance.id}`);
|
||||
expect(job).not.toBeNull();
|
||||
expect(job?.job).toBe("schedule.triggerScheduledTask");
|
||||
}
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"should handle recovery gracefully when no schedules exist in environment",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const mockDevConnectedHandler = vi.fn().mockResolvedValue(true);
|
||||
const triggerCalls: TriggerScheduledTaskParams[] = [];
|
||||
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: {
|
||||
concurrency: 1,
|
||||
disabled: true, // Disable worker to prevent automatic execution
|
||||
pollIntervalMs: 1000,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async (params) => {
|
||||
triggerCalls.push(params);
|
||||
return { success: true };
|
||||
},
|
||||
isDevEnvironmentConnectedHandler: mockDevConnectedHandler,
|
||||
});
|
||||
|
||||
try {
|
||||
// Create test data but no schedules
|
||||
const organization = await prisma.organization.create({
|
||||
data: {
|
||||
title: "Empty Test Org",
|
||||
slug: "empty-test-org",
|
||||
},
|
||||
});
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Empty Test Project",
|
||||
slug: "empty-test-project",
|
||||
externalRef: "empty-test-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "empty-test-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_empty_test_1234",
|
||||
pkApiKey: "pk_empty_test_1234",
|
||||
shortcode: "empty-test-short",
|
||||
},
|
||||
});
|
||||
|
||||
// Perform recovery on empty environment - should not throw errors
|
||||
await expect(
|
||||
engine.recoverSchedulesInEnvironment(project.id, environment.id)
|
||||
).resolves.not.toThrow();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// External-caller backward-compat. Deploy sync (`syncDeclarativeSchedules`)
|
||||
// and schedule upsert both call `registerNextTaskScheduleInstance` with no
|
||||
// `lastScheduleTime`. They run on every app deploy and on every cron edit.
|
||||
// For an existing-and-firing schedule, the call must NOT clobber the
|
||||
// worker payload's `lastScheduleTime` with `undefined` — otherwise the
|
||||
// next fire would surface a stale frozen DB-column value to the customer
|
||||
// (since this PR stops writing that column). The function must derive a
|
||||
// sensible `lastScheduleTime` from the cron expression's previous slot
|
||||
// when the caller doesn't pass one.
|
||||
containerTest(
|
||||
"should derive lastScheduleTime from cron when external callers omit it",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async () => ({ success: true }),
|
||||
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
try {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: "External Caller Org", slug: "external-caller-org" },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "External Caller Project",
|
||||
slug: "external-caller-project",
|
||||
externalRef: "external-caller-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "external-caller-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_external_1234",
|
||||
pkApiKey: "pk_external_1234",
|
||||
shortcode: "external-short",
|
||||
},
|
||||
});
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_external_caller",
|
||||
taskIdentifier: "external-caller-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "external-caller-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
generatorExpression: "*/5 * * * *",
|
||||
generatorDescription: "Every 5 minutes",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Backdate the instance so the cron's previous slot postdates
|
||||
// createdAt — this simulates a long-running schedule, the case
|
||||
// Devin flagged (deploy clobbers lastScheduleTime, post-deploy fire
|
||||
// would otherwise read from a frozen DB column).
|
||||
const longAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
await prisma.taskScheduleInstance.update({
|
||||
where: { id: scheduleInstance.id },
|
||||
data: { createdAt: longAgo },
|
||||
});
|
||||
|
||||
// External-caller pattern — no lastScheduleTime.
|
||||
await engine.registerNextTaskScheduleInstance({
|
||||
instanceId: scheduleInstance.id,
|
||||
});
|
||||
|
||||
const job = await engine.getJob(`scheduled-task-instance:${scheduleInstance.id}`);
|
||||
expect(job).not.toBeNull();
|
||||
// The function should have derived lastScheduleTime from cron,
|
||||
// putting a real timestamp into the worker payload rather than
|
||||
// undefined. The Redis worker stores payloads as JSON, so the value
|
||||
// is a string when read back here — Zod re-coerces it to Date on
|
||||
// dequeue (workerCatalog uses `z.coerce.date()`).
|
||||
const enqueuedLastScheduleTime = (job!.item as { lastScheduleTime?: string })
|
||||
.lastScheduleTime;
|
||||
expect(enqueuedLastScheduleTime).toBeDefined();
|
||||
const derived = new Date(enqueuedLastScheduleTime!);
|
||||
// The derived value should match the cron's previous slot — for
|
||||
// `*/5 * * * *`, a 5-minute boundary in the recent past.
|
||||
expect(derived.getTime()).toBeLessThan(Date.now());
|
||||
expect(derived.getUTCSeconds()).toBe(0);
|
||||
expect(derived.getUTCMinutes() % 5).toBe(0);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Brand-new schedules must NOT receive a cron-derived lastScheduleTime —
|
||||
// the cron's previous slot predates the instance, so it's not a real
|
||||
// previous fire. The first-run sentinel (`if (!payload.lastTimestamp)`)
|
||||
// must keep working.
|
||||
containerTest(
|
||||
"should leave lastScheduleTime undefined for brand-new schedules",
|
||||
{ timeout: 30_000 },
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = new ScheduleEngine({
|
||||
prisma,
|
||||
redis: redisOptions,
|
||||
distributionWindow: { seconds: 10 },
|
||||
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
onTriggerScheduledTask: async () => ({ success: true }),
|
||||
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
try {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: "Brand New Org", slug: "brand-new-org" },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: "Brand New Project",
|
||||
slug: "brand-new-project",
|
||||
externalRef: "brand-new-ref",
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
slug: "brand-new-env",
|
||||
type: "PRODUCTION",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: "tr_brandnew_1234",
|
||||
pkApiKey: "pk_brandnew_1234",
|
||||
shortcode: "brandnew-short",
|
||||
},
|
||||
});
|
||||
const taskSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: "sched_brand_new",
|
||||
taskIdentifier: "brand-new-task",
|
||||
projectId: project.id,
|
||||
deduplicationKey: "brand-new-dedup",
|
||||
userProvidedDeduplicationKey: false,
|
||||
// Hourly cron — the previous slot is plausibly minutes-to-an-hour
|
||||
// ago, comfortably predating an instance just created.
|
||||
generatorExpression: "0 * * * *",
|
||||
generatorDescription: "Hourly",
|
||||
timezone: "UTC",
|
||||
type: "DECLARATIVE",
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
const scheduleInstance = await prisma.taskScheduleInstance.create({
|
||||
data: {
|
||||
taskScheduleId: taskSchedule.id,
|
||||
environmentId: environment.id,
|
||||
projectId: project.id,
|
||||
active: true,
|
||||
},
|
||||
});
|
||||
|
||||
await engine.registerNextTaskScheduleInstance({
|
||||
instanceId: scheduleInstance.id,
|
||||
});
|
||||
|
||||
const job = await engine.getJob(`scheduled-task-instance:${scheduleInstance.id}`);
|
||||
expect(job).not.toBeNull();
|
||||
const enqueuedLastScheduleTime = (job!.item as { lastScheduleTime?: Date })
|
||||
.lastScheduleTime;
|
||||
// Brand-new schedule: cron's previous slot predates instance.createdAt,
|
||||
// so the function leaves lastScheduleTime undefined — the first fire
|
||||
// will report `payload.lastTimestamp: undefined` and customer first-run
|
||||
// sentinel patterns keep working.
|
||||
expect(enqueuedLastScheduleTime).toBeUndefined();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
import { vi } from "vitest";
|
||||
|
||||
// Set extended timeout for container tests
|
||||
vi.setConfig({ testTimeout: 60_000 });
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"outDir": "dist",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"declaration": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "Node16",
|
||||
"module": "Node16",
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.test.ts"],
|
||||
"references": [{ "path": "./tsconfig.src.json" }],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"types": ["vitest/globals"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
sequence: { sequencer: DurationShardingSequencer },
|
||||
globals: true,
|
||||
// CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
|
||||
retry: process.env.CI ? 2 : 0,
|
||||
environment: "node",
|
||||
setupFiles: ["./test/setup.ts"],
|
||||
testTimeout: 30000,
|
||||
hookTimeout: 30000,
|
||||
},
|
||||
esbuild: {
|
||||
target: "node18",
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user