chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,115 @@
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
|
||||
import { z } from "zod";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getRunsReplicationGlobal } from "~/services/runsReplicationGlobal.server";
|
||||
import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
|
||||
// Reference-hold the sessions-replication singleton so module evaluation runs
|
||||
// its initializer (creates the ClickHouse client, subscribes to the logical
|
||||
// replication slot, wires signal handlers) when the webapp boots.
|
||||
//
|
||||
// IMPORTANT: do NOT replace with `void sessionsReplicationInstance;`. With
|
||||
// `"sideEffects": false` in apps/webapp/package.json, esbuild treats `void X;`
|
||||
// as a pure expression statement and eliminates the import — the singleton
|
||||
// initializer never fires. Assignment to globalThis is an observable side
|
||||
// effect the bundler must preserve. See TRI-9864.
|
||||
import { sessionsReplicationInstance } from "~/services/sessionsReplicationInstance.server";
|
||||
(globalThis as Record<string, unknown>).__sessionsReplicationInstance = sessionsReplicationInstance;
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { tracer } from "../tracer.server";
|
||||
import { $replica } from "~/db.server";
|
||||
import { RunsBackfillerService } from "../../services/runsBackfiller.server";
|
||||
|
||||
function initializeWorker() {
|
||||
const redisOptions = {
|
||||
keyPrefix: "admin:worker:",
|
||||
host: env.ADMIN_WORKER_REDIS_HOST,
|
||||
port: env.ADMIN_WORKER_REDIS_PORT,
|
||||
username: env.ADMIN_WORKER_REDIS_USERNAME,
|
||||
password: env.ADMIN_WORKER_REDIS_PASSWORD,
|
||||
enableAutoPipelining: true,
|
||||
...(env.ADMIN_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
};
|
||||
|
||||
logger.debug(`👨🏭 Initializing admin worker at host ${env.ADMIN_WORKER_REDIS_HOST}`);
|
||||
|
||||
const worker = new RedisWorker({
|
||||
name: "admin-worker",
|
||||
redisOptions,
|
||||
catalog: {
|
||||
"admin.backfillRunsToReplication": {
|
||||
schema: z.object({
|
||||
from: z.coerce.date(),
|
||||
to: z.coerce.date(),
|
||||
cursor: z.string().optional(),
|
||||
batchSize: z.coerce.number().int().default(500),
|
||||
delayIntervalMs: z.coerce.number().int().default(1000),
|
||||
}),
|
||||
visibilityTimeoutMs: 60_000 * 15, // 15 minutes
|
||||
retry: {
|
||||
maxAttempts: 5,
|
||||
},
|
||||
},
|
||||
},
|
||||
concurrency: {
|
||||
workers: env.ADMIN_WORKER_CONCURRENCY_WORKERS,
|
||||
tasksPerWorker: env.ADMIN_WORKER_CONCURRENCY_TASKS_PER_WORKER,
|
||||
limit: env.ADMIN_WORKER_CONCURRENCY_LIMIT,
|
||||
},
|
||||
pollIntervalMs: env.ADMIN_WORKER_POLL_INTERVAL,
|
||||
immediatePollIntervalMs: env.ADMIN_WORKER_IMMEDIATE_POLL_INTERVAL,
|
||||
shutdownTimeoutMs: env.ADMIN_WORKER_SHUTDOWN_TIMEOUT_MS,
|
||||
logger: new Logger("AdminWorker", env.ADMIN_WORKER_LOG_LEVEL),
|
||||
jobs: {
|
||||
"admin.backfillRunsToReplication": async ({ payload, id }) => {
|
||||
const replicationService = getRunsReplicationGlobal() ?? runsReplicationInstance;
|
||||
if (!replicationService) {
|
||||
logger.error("Runs replication instance not found");
|
||||
return;
|
||||
}
|
||||
|
||||
const service = new RunsBackfillerService({
|
||||
prisma: $replica,
|
||||
runsReplicationInstance: replicationService,
|
||||
tracer: tracer,
|
||||
});
|
||||
|
||||
const cursor = await service.call({
|
||||
from: payload.from,
|
||||
to: payload.to,
|
||||
cursor: payload.cursor,
|
||||
batchSize: payload.batchSize,
|
||||
});
|
||||
|
||||
if (cursor) {
|
||||
await worker.enqueue({
|
||||
job: "admin.backfillRunsToReplication",
|
||||
payload: {
|
||||
from: payload.from,
|
||||
to: payload.to,
|
||||
cursor,
|
||||
batchSize: payload.batchSize,
|
||||
delayIntervalMs: payload.delayIntervalMs,
|
||||
},
|
||||
id,
|
||||
availableAt: new Date(Date.now() + payload.delayIntervalMs),
|
||||
cancellationKey: id,
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (env.ADMIN_WORKER_ENABLED === "true") {
|
||||
logger.debug(
|
||||
`👨🏭 Starting admin worker at host ${env.ADMIN_WORKER_REDIS_HOST}, pollInterval = ${env.ADMIN_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.ADMIN_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.ADMIN_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.ADMIN_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.ADMIN_WORKER_CONCURRENCY_LIMIT}`
|
||||
);
|
||||
|
||||
worker.start();
|
||||
}
|
||||
|
||||
return worker;
|
||||
}
|
||||
|
||||
export const adminWorker = singleton("adminWorker", initializeWorker);
|
||||
@@ -0,0 +1,655 @@
|
||||
import { openai } from "@ai-sdk/openai";
|
||||
import {
|
||||
parseTSQLSelect,
|
||||
validateQuery,
|
||||
type TableSchema,
|
||||
type ValidationIssue,
|
||||
} from "@internal/tsql";
|
||||
import { streamText, stepCountIs, type LanguageModel, tool } from "ai";
|
||||
import { z } from "zod";
|
||||
import type { AITimeFilter } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/types";
|
||||
|
||||
// Re-export for backwards compatibility
|
||||
export type { AITimeFilter };
|
||||
|
||||
/**
|
||||
* Stream event types for AI query generation
|
||||
*/
|
||||
export type AIQueryStreamEvent =
|
||||
| { type: "thinking"; content: string }
|
||||
| { type: "tool_call"; tool: string; args: unknown }
|
||||
| { type: "tool_result"; tool: string; result: unknown }
|
||||
| { type: "time_filter"; filter: AITimeFilter }
|
||||
| { type: "result"; success: true; query: string; timeFilter?: AITimeFilter }
|
||||
| { type: "result"; success: false; error: string };
|
||||
|
||||
/**
|
||||
* Result type for non-streaming call
|
||||
*/
|
||||
export type AIQueryResult =
|
||||
| { success: true; query: string; timeFilter?: AITimeFilter }
|
||||
| { success: false; error: string };
|
||||
|
||||
/**
|
||||
* Options for query generation
|
||||
*/
|
||||
export interface AIQueryOptions {
|
||||
mode?: "new" | "edit";
|
||||
currentQuery?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation result from the validateTSQLQuery tool
|
||||
*/
|
||||
interface QueryValidationResult {
|
||||
valid: boolean;
|
||||
syntaxError?: string;
|
||||
issues: ValidationIssue[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Service for generating TSQL queries from natural language using AI
|
||||
*/
|
||||
export class AIQueryService {
|
||||
private pendingTimeFilter: AITimeFilter | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly tableSchema: TableSchema[],
|
||||
private readonly model: LanguageModel = openai("gpt-4.1-mini")
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Build the setTimeFilter tool definition
|
||||
* Used by both streamQuery() and call() to keep behavior consistent
|
||||
*/
|
||||
private buildSetTimeFilterTool() {
|
||||
return tool({
|
||||
description:
|
||||
"Set the time filter for the query page UI instead of adding time conditions to the query. ALWAYS use this tool when the user wants to filter by time (e.g., 'last 7 days', 'past hour', 'yesterday'). The UI will apply this filter automatically using the table's time column (triggered_at for runs, bucket_start for metrics). Do NOT add triggered_at or bucket_start to the WHERE clause for time filtering - use this tool instead.",
|
||||
inputSchema: z.object({
|
||||
period: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"Relative time period like '1m', '5m', '30m', '1h', '6h', '12h', '1d', '3d', '7d', '14d', '30d', '90d'. Use this for 'last X days/hours/minutes' requests."
|
||||
),
|
||||
from: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"ISO 8601 timestamp for the start of an absolute date range. Use with 'to' for specific date ranges."
|
||||
),
|
||||
to: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"ISO 8601 timestamp for the end of an absolute date range. Use with 'from' for specific date ranges."
|
||||
),
|
||||
}),
|
||||
execute: async ({ period, from, to }) => {
|
||||
// Store the time filter so we can include it in the result
|
||||
this.pendingTimeFilter = { period, from, to };
|
||||
return {
|
||||
success: true,
|
||||
message: period
|
||||
? `Time filter set to: last ${period}`
|
||||
: `Time filter set to: ${from ?? "start"} - ${to ?? "now"}`,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a TSQL query from natural language, streaming the result
|
||||
*/
|
||||
streamQuery(prompt: string, options: AIQueryOptions = {}) {
|
||||
const { mode = "new", currentQuery } = options;
|
||||
// Reset pending time filter for new request
|
||||
this.pendingTimeFilter = undefined;
|
||||
|
||||
const schemaDescription = this.buildSchemaDescription();
|
||||
const systemPrompt =
|
||||
mode === "edit" && currentQuery
|
||||
? this.buildEditSystemPrompt(schemaDescription)
|
||||
: this.buildSystemPrompt(schemaDescription);
|
||||
|
||||
// Build the user prompt based on mode
|
||||
const userPrompt =
|
||||
mode === "edit" && currentQuery ? this.buildEditUserPrompt(prompt, currentQuery) : prompt;
|
||||
|
||||
return streamText({
|
||||
model: this.model,
|
||||
system: systemPrompt,
|
||||
prompt: userPrompt,
|
||||
tools: {
|
||||
validateTSQLQuery: tool({
|
||||
description:
|
||||
"Validate a TSQL query for syntax errors and schema compliance. Always use this tool to verify your query before returning it to the user.",
|
||||
inputSchema: z.object({
|
||||
query: z.string().describe("The TSQL query to validate"),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
return this.validateQuery(query);
|
||||
},
|
||||
}),
|
||||
getTableSchema: tool({
|
||||
description:
|
||||
"Get detailed schema information about available tables and columns. Use this to understand what data is available and how to query it.",
|
||||
inputSchema: z.object({
|
||||
tableName: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Optional: specific table name to get details for"),
|
||||
}),
|
||||
execute: async ({ tableName }) => {
|
||||
return this.getSchemaInfo(tableName);
|
||||
},
|
||||
}),
|
||||
setTimeFilter: this.buildSetTimeFilterTool(),
|
||||
},
|
||||
stopWhen: stepCountIs(5),
|
||||
experimental_telemetry: {
|
||||
isEnabled: true,
|
||||
metadata: {
|
||||
feature: "ai-query-generator",
|
||||
mode,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the pending time filter (set by the AI during query generation)
|
||||
*/
|
||||
getPendingTimeFilter(): AITimeFilter | undefined {
|
||||
return this.pendingTimeFilter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a TSQL query from natural language (non-streaming)
|
||||
*/
|
||||
async call(prompt: string, options: AIQueryOptions = {}): Promise<AIQueryResult> {
|
||||
const { mode = "new", currentQuery } = options;
|
||||
// Reset pending time filter for new request
|
||||
this.pendingTimeFilter = undefined;
|
||||
|
||||
const schemaDescription = this.buildSchemaDescription();
|
||||
const systemPrompt =
|
||||
mode === "edit" && currentQuery
|
||||
? this.buildEditSystemPrompt(schemaDescription)
|
||||
: this.buildSystemPrompt(schemaDescription);
|
||||
|
||||
// Build the user prompt based on mode
|
||||
const userPrompt =
|
||||
mode === "edit" && currentQuery ? this.buildEditUserPrompt(prompt, currentQuery) : prompt;
|
||||
|
||||
const result = await streamText({
|
||||
model: this.model,
|
||||
system: systemPrompt,
|
||||
prompt: userPrompt,
|
||||
tools: {
|
||||
validateTSQLQuery: tool({
|
||||
description:
|
||||
"Validate a TSQL query for syntax errors and schema compliance. Always use this tool to verify your query before returning it to the user.",
|
||||
inputSchema: z.object({
|
||||
query: z.string().describe("The TSQL query to validate"),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
return this.validateQuery(query);
|
||||
},
|
||||
}),
|
||||
getTableSchema: tool({
|
||||
description:
|
||||
"Get detailed schema information about available tables and columns. Use this to understand what data is available and how to query it.",
|
||||
inputSchema: z.object({
|
||||
tableName: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Optional: specific table name to get details for"),
|
||||
}),
|
||||
execute: async ({ tableName }) => {
|
||||
return this.getSchemaInfo(tableName);
|
||||
},
|
||||
}),
|
||||
setTimeFilter: this.buildSetTimeFilterTool(),
|
||||
},
|
||||
stopWhen: stepCountIs(5),
|
||||
experimental_telemetry: {
|
||||
isEnabled: true,
|
||||
metadata: {
|
||||
feature: "ai-query-generator",
|
||||
mode,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Wait for the full response
|
||||
const text = await result.text;
|
||||
|
||||
// Try to extract a valid query from the response
|
||||
const query = this.extractQueryFromResponse(text);
|
||||
|
||||
if (query) {
|
||||
// Validate the extracted query one more time
|
||||
const validation = this.validateQuery(query);
|
||||
if (validation.valid) {
|
||||
return { success: true, query, timeFilter: this.pendingTimeFilter };
|
||||
} else {
|
||||
const errorMessages = validation.issues.map((i) => i.message).join("; ");
|
||||
return {
|
||||
success: false,
|
||||
error: validation.syntaxError || errorMessages || "Query validation failed",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// If no query was found, check if there's an error message
|
||||
if (text.toLowerCase().includes("cannot") || text.toLowerCase().includes("unable")) {
|
||||
return { success: false, error: text.slice(0, 200) };
|
||||
}
|
||||
|
||||
return { success: false, error: "Could not generate a valid query" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a TSQL query using the parser and validator
|
||||
*/
|
||||
private validateQuery(query: string): QueryValidationResult {
|
||||
try {
|
||||
// First, try to parse the query
|
||||
const ast = parseTSQLSelect(query);
|
||||
|
||||
// Then validate against the schema
|
||||
const validationResult = validateQuery(ast, this.tableSchema);
|
||||
|
||||
return {
|
||||
valid: validationResult.valid,
|
||||
issues: validationResult.issues,
|
||||
};
|
||||
} catch (error) {
|
||||
// Syntax error during parsing
|
||||
return {
|
||||
valid: false,
|
||||
syntaxError: error instanceof Error ? error.message : String(error),
|
||||
issues: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get schema information for the AI
|
||||
*/
|
||||
private getSchemaInfo(tableName?: string): {
|
||||
tables: Array<{
|
||||
name: string;
|
||||
description?: string;
|
||||
columns: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
description?: string;
|
||||
allowedValues?: string[];
|
||||
example?: string;
|
||||
}>;
|
||||
}>;
|
||||
} {
|
||||
const tables = tableName
|
||||
? this.tableSchema.filter((t) => t.name.toLowerCase() === tableName?.toLowerCase())
|
||||
: this.tableSchema;
|
||||
|
||||
return {
|
||||
tables: tables.map((table) => ({
|
||||
name: table.name,
|
||||
description: table.description,
|
||||
columns: Object.values(table.columns).map((col) => ({
|
||||
name: col.name,
|
||||
type: col.type,
|
||||
description: col.description,
|
||||
allowedValues: col.valueMap ? Object.values(col.valueMap) : col.allowedValues,
|
||||
example: col.example,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a description of the schema for the system prompt
|
||||
*/
|
||||
private buildSchemaDescription(): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
for (const table of this.tableSchema) {
|
||||
parts.push(`## Table: ${table.name}`);
|
||||
if (table.description) {
|
||||
parts.push(table.description);
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
// Identify core columns
|
||||
const coreColumns = Object.values(table.columns)
|
||||
.filter((col) => col.coreColumn === true)
|
||||
.map((col) => col.name);
|
||||
if (coreColumns.length > 0) {
|
||||
parts.push(`Core columns (use these as defaults): ${coreColumns.join(", ")}`);
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
parts.push("Columns:");
|
||||
|
||||
for (const col of Object.values(table.columns)) {
|
||||
let colDesc = `- ${col.name} (${col.type})`;
|
||||
if (col.coreColumn) {
|
||||
colDesc += " [CORE]";
|
||||
}
|
||||
if (col.description) {
|
||||
colDesc += `: ${col.description}`;
|
||||
}
|
||||
parts.push(colDesc);
|
||||
|
||||
// Add allowed values for enum-like columns
|
||||
const allowedValues = col.valueMap ? Object.values(col.valueMap) : col.allowedValues;
|
||||
if (allowedValues && allowedValues.length > 0 && allowedValues.length <= 20) {
|
||||
parts.push(` Allowed values: ${allowedValues.join(", ")}`);
|
||||
}
|
||||
|
||||
// Add example if available
|
||||
if (col.example) {
|
||||
parts.push(` Example: ${col.example}`);
|
||||
}
|
||||
}
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system prompt for the AI
|
||||
*/
|
||||
private buildSystemPrompt(schemaDescription: string): string {
|
||||
return `You are an expert SQL assistant that generates TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
|
||||
|
||||
## Your Task
|
||||
Convert natural language requests into valid TSQL SELECT queries. Always validate your queries using the validateTSQLQuery tool before returning them.
|
||||
|
||||
## Available Schema
|
||||
${schemaDescription}
|
||||
|
||||
## Choosing the Right Table
|
||||
|
||||
- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
|
||||
- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
|
||||
|
||||
When the user mentions "CPU", "memory", "utilization", "resource usage", or similar terms, query the \`metrics\` table. When they mention "runs", "tasks", "failures", "status", "duration", or "cost", query the \`runs\` table.
|
||||
|
||||
## TSQL Syntax Guide
|
||||
|
||||
TSQL supports standard SQL syntax with some ClickHouse-specific features:
|
||||
|
||||
### Basic SELECT
|
||||
\`\`\`sql
|
||||
SELECT column1, column2, ...
|
||||
FROM table_name
|
||||
WHERE conditions
|
||||
ORDER BY column [ASC|DESC]
|
||||
LIMIT n
|
||||
\`\`\`
|
||||
|
||||
### Filtering (WHERE clause)
|
||||
- Comparison: =, !=, <, >, <=, >=
|
||||
- Logical: AND, OR, NOT
|
||||
- Pattern matching: LIKE, ILIKE (case-insensitive), NOT LIKE
|
||||
- Range: BETWEEN value1 AND value2
|
||||
- Set membership: IN ('value1', 'value2'), NOT IN (...)
|
||||
- Null checks: IS NULL, IS NOT NULL
|
||||
- Array contains: has(array_column, 'value')
|
||||
|
||||
### Aggregations
|
||||
- count() - count rows
|
||||
- countIf(condition) - count rows matching condition
|
||||
- sum(column), sumIf(column, condition)
|
||||
- avg(column), min(column), max(column)
|
||||
- uniq(column) - approximate unique count
|
||||
- quantile(p)(column) - percentile (p between 0 and 1)
|
||||
- groupArray(column) - collect values into array
|
||||
|
||||
### Grouping
|
||||
\`\`\`sql
|
||||
SELECT column, count() as cnt
|
||||
FROM table
|
||||
GROUP BY column
|
||||
HAVING cnt > 10
|
||||
\`\`\`
|
||||
|
||||
### Date/Time Functions
|
||||
- timeBucket() - automatically bucket by time. Uses the table's time column and picks the best interval based on the query's time range. Use in SELECT and reference as \`timeBucket\` in GROUP BY / ORDER BY.
|
||||
- now() - current timestamp
|
||||
- today() - current date
|
||||
- toDate(datetime) - extract date
|
||||
- toStartOfDay/Hour/Minute(datetime)
|
||||
- dateDiff('unit', start, end) - difference in units (second, minute, hour, day, week, month, year)
|
||||
- INTERVAL n unit - time interval (e.g., INTERVAL 7 DAY)
|
||||
|
||||
### Time Bucketing
|
||||
When the user wants to see data "over time", "by hour", "by day", or any time-series aggregation, prefer \`timeBucket()\` over manual \`toStartOfHour\`/\`toStartOfDay\` calls. \`timeBucket()\` automatically picks the right interval for the current time range.
|
||||
|
||||
\`\`\`sql
|
||||
-- Runs over time (bucket size auto-selected)
|
||||
SELECT timeBucket(), count() AS run_count
|
||||
FROM runs
|
||||
GROUP BY timeBucket
|
||||
ORDER BY timeBucket
|
||||
LIMIT 1000
|
||||
\`\`\`
|
||||
|
||||
Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifically requests a particular bucket size (e.g., "group by hour", "bucket by day").
|
||||
|
||||
### Common Patterns
|
||||
|
||||
#### Runs table
|
||||
- Status filter: WHERE status = 'Failed' or WHERE status IN ('Failed', 'Crashed')
|
||||
- Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at/bucket_start in WHERE clause)
|
||||
|
||||
#### Metrics table
|
||||
- Filter by metric name: WHERE metric_name = 'process.cpu.utilization'
|
||||
- Filter by run: WHERE run_id = 'run_abc123'
|
||||
- Filter by task: WHERE task_identifier = 'my-task'
|
||||
- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p95, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
|
||||
- Use \`metric_value\` — the metric's observed value
|
||||
- Use prettyFormat(expr, 'bytes') to tell the UI to format values as bytes (e.g., "1.50 GiB") — keeps values numeric for charts
|
||||
- Use prettyFormat(expr, 'percent') for percentage values
|
||||
- prettyFormat does NOT change the SQL — it only adds a display hint
|
||||
- Available format types: bytes, decimalBytes, percent, quantity, duration, durationSeconds, costInDollars
|
||||
- For memory metrics (including nodejs.heap.*), always use prettyFormat with 'bytes'
|
||||
- For CPU utilization, consider prettyFormat with 'percent'
|
||||
|
||||
\`\`\`sql
|
||||
-- CPU utilization over time for a task
|
||||
SELECT timeBucket(), task_identifier, prettyFormat(avg(metric_value), 'percent') AS avg_cpu
|
||||
FROM metrics
|
||||
WHERE metric_name = 'process.cpu.utilization'
|
||||
GROUP BY timeBucket, task_identifier
|
||||
ORDER BY timeBucket
|
||||
LIMIT 1000
|
||||
\`\`\`
|
||||
|
||||
\`\`\`sql
|
||||
-- Peak memory usage per run
|
||||
SELECT run_id, task_identifier, prettyFormat(max(metric_value), 'bytes') AS peak_memory
|
||||
FROM metrics
|
||||
WHERE metric_name = 'process.memory.usage'
|
||||
GROUP BY run_id, task_identifier
|
||||
ORDER BY peak_memory DESC
|
||||
LIMIT 100
|
||||
\`\`\`
|
||||
|
||||
## Important Rules
|
||||
|
||||
1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
|
||||
2. Always select only the specific columns needed for the request
|
||||
3. When column selection is ambiguous, use the core columns marked [CORE] in the schema
|
||||
4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding time conditions to the WHERE clause. The UI has a time filter that will apply this automatically. This applies to both the \`runs\` table (triggered_at) and the \`metrics\` table (bucket_start).
|
||||
5. Do NOT add \`triggered_at\` or \`bucket_start\` to WHERE clauses for time filtering - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
|
||||
6. **TIME BUCKETING**: When the user wants to see data over time or in time buckets, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
|
||||
7. ALWAYS use the validateTSQLQuery tool to check your query before returning it
|
||||
8. If validation fails, fix the issues and try again (up to 3 attempts)
|
||||
9. Use column names exactly as defined in the schema (case-sensitive)
|
||||
10. For enum columns like status, use the allowed values shown in the schema
|
||||
11. Always include a LIMIT clause (default to 100 if not specified)
|
||||
12. Use meaningful column aliases with AS for aggregations
|
||||
13. Format queries with proper indentation for readability
|
||||
|
||||
## Response Format
|
||||
|
||||
After validating successfully, return ONLY the SQL query wrapped in a code block:
|
||||
|
||||
\`\`\`sql
|
||||
SELECT ...
|
||||
FROM ...
|
||||
\`\`\`
|
||||
|
||||
If you cannot generate a valid query, explain why briefly.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system prompt for edit mode
|
||||
*/
|
||||
private buildEditSystemPrompt(schemaDescription: string): string {
|
||||
return `You are an expert SQL assistant that modifies existing TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
|
||||
|
||||
## Your Task
|
||||
Modify the provided TSQL query according to the user's instructions. Make only the changes requested - preserve the existing query structure where possible.
|
||||
|
||||
## Available Schema
|
||||
${schemaDescription}
|
||||
|
||||
## Choosing the Right Table
|
||||
|
||||
- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
|
||||
- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
|
||||
|
||||
## TSQL Syntax Guide
|
||||
|
||||
TSQL supports standard SQL syntax with some ClickHouse-specific features:
|
||||
|
||||
### Basic SELECT
|
||||
\`\`\`sql
|
||||
SELECT column1, column2, ...
|
||||
FROM table_name
|
||||
WHERE conditions
|
||||
ORDER BY column [ASC|DESC]
|
||||
LIMIT n
|
||||
\`\`\`
|
||||
|
||||
### Filtering (WHERE clause)
|
||||
- Comparison: =, !=, <, >, <=, >=
|
||||
- Logical: AND, OR, NOT
|
||||
- Pattern matching: LIKE, ILIKE (case-insensitive), NOT LIKE
|
||||
- Range: BETWEEN value1 AND value2
|
||||
- Set membership: IN ('value1', 'value2'), NOT IN (...)
|
||||
- Null checks: IS NULL, IS NOT NULL
|
||||
- Array contains: has(array_column, 'value')
|
||||
|
||||
### Aggregations
|
||||
- count() - count rows
|
||||
- countIf(condition) - count rows matching condition
|
||||
- sum(column), sumIf(column, condition)
|
||||
- avg(column), min(column), max(column)
|
||||
- uniq(column) - approximate unique count
|
||||
- quantile(p)(column) - percentile (p between 0 and 1)
|
||||
- groupArray(column) - collect values into array
|
||||
|
||||
### Grouping
|
||||
\`\`\`sql
|
||||
SELECT column, count() as cnt
|
||||
FROM table
|
||||
GROUP BY column
|
||||
HAVING cnt > 10
|
||||
\`\`\`
|
||||
|
||||
### Date/Time Functions
|
||||
- timeBucket() - automatically bucket by time. Uses the table's time column and picks the best interval based on the query's time range. Use in SELECT and reference as \`timeBucket\` in GROUP BY / ORDER BY.
|
||||
- now() - current timestamp
|
||||
- today() - current date
|
||||
- toDate(datetime) - extract date
|
||||
- toStartOfDay/Hour/Minute(datetime)
|
||||
- dateDiff('unit', start, end) - difference in units (second, minute, hour, day, week, month, year)
|
||||
- INTERVAL n unit - time interval (e.g., INTERVAL 7 DAY)
|
||||
|
||||
### Time Bucketing
|
||||
When the user wants to see data "over time", "by hour", "by day", or any time-series aggregation, prefer \`timeBucket()\` over manual \`toStartOfHour\`/\`toStartOfDay\` calls unless the user specifically requests a particular bucket size.
|
||||
|
||||
\`\`\`sql
|
||||
SELECT timeBucket(), count() AS run_count
|
||||
FROM runs
|
||||
GROUP BY timeBucket
|
||||
ORDER BY timeBucket
|
||||
LIMIT 1000
|
||||
\`\`\`
|
||||
|
||||
### Common Metrics Patterns
|
||||
- Filter by metric: WHERE metric_name = 'process.cpu.utilization'
|
||||
- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p50, nodejs.event_loop.delay.p99, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
|
||||
- Use \`metric_value\` — the metric's observed value
|
||||
- Use prettyFormat(expr, 'bytes') for memory metrics (including nodejs.heap.*), prettyFormat(expr, 'percent') for CPU utilization
|
||||
- prettyFormat does NOT change the SQL — it only adds a display hint for the UI
|
||||
|
||||
## Important Rules
|
||||
|
||||
1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
|
||||
2. If the existing query uses SELECT *, replace it with specific columns (use core columns marked [CORE] as defaults)
|
||||
3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying time column conditions. If the existing query has \`triggered_at\` or \`bucket_start\` in WHERE for time filtering, consider removing it and using \`setTimeFilter\` instead.
|
||||
4. **TIME BUCKETING**: When adding time-series grouping, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
|
||||
5. ALWAYS use the validateTSQLQuery tool to check your modified query before returning it
|
||||
6. If validation fails, fix the issues and try again (up to 3 attempts)
|
||||
7. Use column names exactly as defined in the schema (case-sensitive)
|
||||
8. For enum columns like status, use the allowed values shown in the schema
|
||||
9. Always include a LIMIT clause (default to 100 if not specified)
|
||||
10. Preserve the user's existing query structure and style where possible
|
||||
11. Only make the changes specifically requested by the user
|
||||
|
||||
## Response Format
|
||||
|
||||
After validating successfully, return ONLY the modified SQL query wrapped in a code block:
|
||||
|
||||
\`\`\`sql
|
||||
SELECT ...
|
||||
FROM ...
|
||||
\`\`\`
|
||||
|
||||
If you cannot make the requested modification, explain why briefly.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the user prompt for edit mode
|
||||
*/
|
||||
private buildEditUserPrompt(userRequest: string, currentQuery: string): string {
|
||||
return `Here is the current TSQL query:
|
||||
|
||||
\`\`\`sql
|
||||
${currentQuery}
|
||||
\`\`\`
|
||||
|
||||
Please modify this query according to the following instructions:
|
||||
|
||||
${userRequest}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a SQL query from the AI response text
|
||||
*/
|
||||
private extractQueryFromResponse(text: string): string | null {
|
||||
// Try to extract from code block first
|
||||
const codeBlockMatch = text.match(/```(?:sql)?\s*([\s\S]*?)```/i);
|
||||
if (codeBlockMatch) {
|
||||
return codeBlockMatch[1].trim();
|
||||
}
|
||||
|
||||
// Try to find a SELECT statement
|
||||
const selectMatch = text.match(/SELECT[\s\S]+?(?:LIMIT\s+\d+|;|$)/i);
|
||||
if (selectMatch) {
|
||||
return selectMatch[0].trim().replace(/;$/, "");
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { openai } from "@ai-sdk/openai";
|
||||
import { generateText, type LanguageModel } from "ai";
|
||||
import { env } from "~/env.server";
|
||||
|
||||
/**
|
||||
* Result type for title generation
|
||||
*/
|
||||
export type AIQueryTitleResult =
|
||||
| { success: true; title: string }
|
||||
| { success: false; error: string };
|
||||
|
||||
/**
|
||||
* Service for generating concise titles for SQL queries using AI
|
||||
*/
|
||||
export class AIQueryTitleService {
|
||||
constructor(private readonly model: LanguageModel = openai("gpt-4o-mini")) {}
|
||||
|
||||
/**
|
||||
* Generate a concise title for a SQL query
|
||||
*/
|
||||
async generateTitle(query: string): Promise<AIQueryTitleResult> {
|
||||
if (!env.OPENAI_API_KEY) {
|
||||
return { success: false, error: "OpenAI API key is not configured" };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await generateText({
|
||||
model: this.model,
|
||||
system: `You are a helpful assistant that generates concise titles for SQL queries.
|
||||
|
||||
Your task is to create a short, descriptive title (5-10 words) that summarizes what the query does.
|
||||
|
||||
Guidelines:
|
||||
- Focus on the main purpose/intent of the query
|
||||
- Use plain language, not technical SQL terms
|
||||
- Start with an action verb when appropriate (e.g., "Count", "List", "Show", "Find")
|
||||
- Be specific about what data is being retrieved
|
||||
- Do not include quotes around the title
|
||||
- Do not include punctuation at the end
|
||||
|
||||
Examples:
|
||||
- "Failed runs by hour over 7 days"
|
||||
- "Top 50 most expensive task runs"
|
||||
- "Run counts grouped by status"
|
||||
- "Average execution time by task"
|
||||
- "Recent runs with errors"`,
|
||||
prompt: `Generate a concise title for this SQL query:\n\n${query}`,
|
||||
maxOutputTokens: 50,
|
||||
experimental_telemetry: {
|
||||
isEnabled: true,
|
||||
metadata: {
|
||||
feature: "ai-query-title",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const title = result.text.trim();
|
||||
|
||||
if (!title) {
|
||||
return { success: false, error: "No title generated" };
|
||||
}
|
||||
|
||||
return { success: true, title };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "Failed to generate title",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
import { openai } from "@ai-sdk/openai";
|
||||
import { type TaskTriggerSource } from "@trigger.dev/database";
|
||||
import { generateText, stepCountIs, type LanguageModel, Output, tool } from "ai";
|
||||
import { z } from "zod";
|
||||
import { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
|
||||
import { logger } from "~/services/logger.server";
|
||||
|
||||
const AIFilters = TaskRunListSearchFilters.omit({
|
||||
environments: true,
|
||||
from: true,
|
||||
to: true,
|
||||
}).extend({
|
||||
from: z.string().optional().describe("The ISO datetime to filter from"),
|
||||
to: z.string().optional().describe("The ISO datetime to filter to"),
|
||||
});
|
||||
|
||||
// The response is wrapped in an object with a single `response` field because
|
||||
// OpenAI structured outputs (both the Chat and Responses APIs) require the root
|
||||
// JSON Schema to be `type: "object"`. A bare `z.discriminatedUnion` compiles to
|
||||
// a root-level `anyOf`, which OpenAI rejects ("schema must be of type object").
|
||||
// Nesting the union under a property keeps the discriminated-union ergonomics
|
||||
// while satisfying the root-object constraint.
|
||||
const AIFilterResponseSchema = z
|
||||
.object({
|
||||
response: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(true),
|
||||
filters: AIFilters,
|
||||
}),
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
error: z.string().describe("A short human-readable error message"),
|
||||
}),
|
||||
]),
|
||||
})
|
||||
.describe("The response from the AI filter service");
|
||||
|
||||
export interface QueryQueues {
|
||||
query(
|
||||
search: string | undefined,
|
||||
type: "task" | "custom" | undefined
|
||||
): Promise<{
|
||||
queues: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface QueryVersions {
|
||||
query(
|
||||
versionPrefix: string | undefined,
|
||||
isCurrent: boolean | undefined
|
||||
): Promise<
|
||||
| {
|
||||
versions: string[];
|
||||
}
|
||||
| {
|
||||
version: string;
|
||||
}
|
||||
>;
|
||||
}
|
||||
|
||||
export interface QueryTags {
|
||||
query(search: string | undefined): Promise<{
|
||||
tags: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface QueryTasks {
|
||||
query(): Promise<{
|
||||
tasks: { slug: string; triggerSource: TaskTriggerSource }[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export type AIFilterResult =
|
||||
| {
|
||||
success: true;
|
||||
filters: TaskRunListSearchFilters;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export class AIRunFilterService {
|
||||
constructor(
|
||||
private readonly queryFns: {
|
||||
queryTags: QueryTags;
|
||||
queryVersions: QueryVersions;
|
||||
queryQueues: QueryQueues;
|
||||
queryTasks: QueryTasks;
|
||||
},
|
||||
private readonly model: LanguageModel = openai("gpt-4o-mini")
|
||||
) {}
|
||||
|
||||
async call(text: string, environmentId: string): Promise<AIFilterResult> {
|
||||
try {
|
||||
const result = await generateText({
|
||||
model: this.model,
|
||||
experimental_output: Output.object({ schema: AIFilterResponseSchema }),
|
||||
// Disable OpenAI strict JSON-schema mode. The filters schema has many
|
||||
// optional fields, and strict mode requires every property to appear in
|
||||
// `required` (it rejects bare optionals). Non-strict mode treats the
|
||||
// schema as guidance; the `AIFilters.safeParse` below still validates
|
||||
// the result, so correctness is preserved.
|
||||
providerOptions: {
|
||||
openai: {
|
||||
strictJsonSchema: false,
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
lookupTags: tool({
|
||||
description: "Look up available tags in the environment",
|
||||
inputSchema: z.object({
|
||||
query: z.string().optional().describe("Optional search query to filter tags"),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
return await this.queryFns.queryTags.query(query);
|
||||
},
|
||||
}),
|
||||
lookupVersions: tool({
|
||||
description:
|
||||
"Look up available versions in the environment. If you specify `isCurrent` it will return a single version string if it finds one. Otherwise it will return an array of version strings.",
|
||||
inputSchema: z.object({
|
||||
isCurrent: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe("If true, only return the current version"),
|
||||
versionPrefix: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"Optional version name to filter (e.g. 20250701.1), it uses contains to compare. Don't pass `latest` or `current`, the query has to be in the reverse date format specified. Leave out to get all recent versions."
|
||||
),
|
||||
}),
|
||||
execute: async ({ versionPrefix, isCurrent }) => {
|
||||
return await this.queryFns.queryVersions.query(versionPrefix, isCurrent);
|
||||
},
|
||||
}),
|
||||
lookupQueues: tool({
|
||||
description: "Look up available queues in the environment",
|
||||
inputSchema: z.object({
|
||||
query: z.string().optional().describe("Optional search query to filter queues"),
|
||||
type: z
|
||||
.enum(["task", "custom"])
|
||||
.optional()
|
||||
.describe(
|
||||
"Filter by queue type, only do this if the user specifies it explicitly."
|
||||
),
|
||||
}),
|
||||
execute: async ({ query, type }) => {
|
||||
return await this.queryFns.queryQueues.query(query, type);
|
||||
},
|
||||
}),
|
||||
lookupTasks: tool({
|
||||
description:
|
||||
"Look up available tasks in the environment. It will return each one. The `slug` is used for the filtering. You also get the triggerSource which is either `STANDARD` or `SCHEDULED`",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
return await this.queryFns.queryTasks.query();
|
||||
},
|
||||
}),
|
||||
},
|
||||
stopWhen: stepCountIs(5),
|
||||
system: `You are an AI assistant that converts natural language descriptions into structured filter parameters for a task run filtering system.
|
||||
|
||||
Available filter options:
|
||||
- statuses: Array of run statuses (PENDING, EXECUTING, COMPLETED_SUCCESSFULLY, COMPLETED_WITH_ERRORS, CANCELED, TIMED_OUT, CRASHED, etc.)
|
||||
- period: Time period string (e.g., "1h", "7d", "30d", "1y")
|
||||
- from/to: ISO date string. Today's date is ${new Date().toISOString()}, if they only specify a day use the current month. If they don't specify a year use the current year. If they don't specify a time of day use midnight.
|
||||
- tags: Array of tag names to filter by. Use the lookupTags tool to get the tags.
|
||||
- tasks: Array of task identifiers to filter by. Use the lookupTasks tool to get the tasks.
|
||||
- machines: Array of machine presets (micro, small, small-2x, medium, large, xlarge, etc.)
|
||||
- queues: Array of queue names to filter by. Use the lookupQueues tool to get the queues.
|
||||
- versions: Array of version identifiers to filter by. Use the lookupVersions tool to get the versions. The "latest" version will be the first returned. The "current" or "deployed" version will have isCurrent set to true.
|
||||
- rootOnly: Boolean to show only root runs (not child runs)
|
||||
- runId: Array of specific run IDs to filter by
|
||||
- batchId: Specific batch ID to filter by
|
||||
- scheduleId: Specific schedule ID to filter by
|
||||
|
||||
|
||||
Common patterns to recognize:
|
||||
- "failed runs" → statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"].
|
||||
- "runs not dequeued yet" → statuses: ["PENDING", "PENDING_VERSION", "DELAYED"]
|
||||
- If they say "only failed" then only use "COMPLETED_WITH_ERRORS".
|
||||
- "successful runs" → statuses: ["COMPLETED_SUCCESSFULLY"]
|
||||
- "running runs" → statuses: ["EXECUTING", "RETRYING_AFTER_FAILURE", "WAITING_TO_RESUME"]
|
||||
- "pending runs" → statuses: ["PENDING", "PENDING_VERSION", "DELAYED"]
|
||||
- "past 7 days" → period: "7d"
|
||||
- "last hour" → period: "1h"
|
||||
- "this month" → period: "30d"
|
||||
- "June 16" -> return a from/to filter.
|
||||
- "with tag X" → tags: ["X"]
|
||||
- "from task Y" → tasks: ["Y"]
|
||||
- "using large machine" → machines: ["large-1x", "large-2x"]
|
||||
- "root only" → rootOnly: true
|
||||
|
||||
Time-specific patterns:
|
||||
- "around 8am today" → from/to: 7am-9am today (1-2 hour window around the time)
|
||||
- "at 2pm yesterday" → from/to: 2pm-2:59pm yesterday (exact hour)
|
||||
- "this morning" → from/to: 6am-12pm today
|
||||
- "this afternoon" → from/to: 12pm-6pm today
|
||||
- "this evening" → from/to: 6pm-10pm today
|
||||
- "started around X" / "that started at X" → same as time filtering (treat "started" as temporal filter)
|
||||
- "began around X" / "ran at X" → same as time filtering
|
||||
|
||||
When handling specific times:
|
||||
- "around" adds ±1 hour buffer (e.g., "around 8am" = 7am-9am)
|
||||
- "at" means exact hour (e.g., "at 2pm" = 2pm-2:59pm)
|
||||
- For relative days: "today" = current date, "yesterday" = current date - 1 day
|
||||
- Convert times to ISO format for from/to filters
|
||||
|
||||
Use the available tools to look up actual tags, versions, queues, and tasks in the environment when the user mentions them. This will help you provide accurate filter values.
|
||||
|
||||
Unless they specify they only want root runs, set rootOnly to false.
|
||||
|
||||
IMPORTANT: Return ONLY the filters that are explicitly mentioned or can be reasonably inferred. If the description is unclear or doesn't match any known patterns, return an empty filters object {} and explain why in the explanation field.
|
||||
|
||||
The filters object should only contain the fields that are actually being filtered. Do not include fields with empty arrays or undefined values.
|
||||
|
||||
CRITICAL: The response must be a valid JSON object with a single top-level "response" key wrapping this structure:
|
||||
{
|
||||
"response": {
|
||||
"success": true,
|
||||
"filters": {
|
||||
// only include fields that have actual values
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
or if you can't figure out the filters then return:
|
||||
{
|
||||
"response": {
|
||||
"success": false,
|
||||
"error": "<short human understandable suggestion>"
|
||||
}
|
||||
}
|
||||
|
||||
Make the error no more than 8 words.
|
||||
`,
|
||||
prompt: text,
|
||||
experimental_telemetry: {
|
||||
isEnabled: true,
|
||||
metadata: {
|
||||
environmentId,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const output = result.experimental_output.response;
|
||||
|
||||
if (!output.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: output.error,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate the filters against the schema to catch any issues
|
||||
const validationResult = AIFilters.safeParse(output.filters);
|
||||
if (!validationResult.success) {
|
||||
logger.error("AI filter validation failed", {
|
||||
errors: validationResult.error.errors,
|
||||
filters: output.filters,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: "AI response validation failed",
|
||||
};
|
||||
}
|
||||
|
||||
// `from`/`to` are validated as strings, so a malformed value (e.g. the
|
||||
// model returning a non-ISO date) would survive safeParse and then
|
||||
// produce NaN here. NaN serializes to `null` over JSON, silently dropping
|
||||
// the date constraint while still reporting success — so reject it.
|
||||
const from = validationResult.data.from
|
||||
? new Date(validationResult.data.from).getTime()
|
||||
: undefined;
|
||||
const to = validationResult.data.to
|
||||
? new Date(validationResult.data.to).getTime()
|
||||
: undefined;
|
||||
|
||||
if ((from !== undefined && Number.isNaN(from)) || (to !== undefined && Number.isNaN(to))) {
|
||||
logger.error("AI filter returned an invalid datetime", {
|
||||
from: validationResult.data.from,
|
||||
to: validationResult.data.to,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: "AI response contained an invalid date",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
filters: {
|
||||
...validationResult.data,
|
||||
from,
|
||||
to,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error("AI filter processing failed", {
|
||||
error,
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
text,
|
||||
environmentId,
|
||||
});
|
||||
|
||||
// If it's a schema validation error, provide more specific feedback
|
||||
if (error instanceof Error && error.message.includes("schema")) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { Ratelimit } from "@upstash/ratelimit";
|
||||
import { type RedisWithClusterOptions } from "~/redis.server";
|
||||
import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
// The query ai-title endpoint lives under `/resources/*`, which the global
|
||||
// apiRateLimiter (only `/api/*`) does not cover, so it needs its own per-user
|
||||
// cap. Exported so the policy is asserted in tests rather than re-encoded.
|
||||
export const AI_TITLE_RATE_LIMIT_ATTEMPTS = 30;
|
||||
export const AI_TITLE_RATE_LIMIT_WINDOW = "10 m" as const;
|
||||
|
||||
/**
|
||||
* Build the ai-title per-user rate limiter. Production uses the env-derived
|
||||
* rate-limit Redis; tests inject a container Redis.
|
||||
*/
|
||||
export function createAITitleRateLimiter(redisOptions?: RedisWithClusterOptions): RateLimiter {
|
||||
return new RateLimiter({
|
||||
...(redisOptions ? { redisClient: createRedisRateLimitClient(redisOptions) } : {}),
|
||||
keyPrefix: "query.ai-title",
|
||||
limiter: Ratelimit.slidingWindow(AI_TITLE_RATE_LIMIT_ATTEMPTS, AI_TITLE_RATE_LIMIT_WINDOW),
|
||||
logFailure: true,
|
||||
});
|
||||
}
|
||||
|
||||
export const aiTitleRateLimiter = singleton("aiTitleRateLimiter", () => createAITitleRateLimiter());
|
||||
@@ -0,0 +1,150 @@
|
||||
import {
|
||||
type ProjectAlertChannel,
|
||||
type ProjectAlertType,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import { nanoid } from "nanoid";
|
||||
import { env } from "~/env.server";
|
||||
import { findProjectByRef } from "~/models/project.server";
|
||||
import { encryptSecret } from "~/services/secrets/secretStore.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
|
||||
import { BaseService, ServiceValidationError } from "../baseService.server";
|
||||
import { assertSafeWebhookUrl, UnsafeWebhookUrlError } from "./safeWebhookUrl.server";
|
||||
|
||||
export type CreateAlertChannelOptions = {
|
||||
name: string;
|
||||
alertTypes: ProjectAlertType[];
|
||||
environmentTypes: RuntimeEnvironmentType[];
|
||||
deduplicationKey?: string;
|
||||
channel:
|
||||
| {
|
||||
type: "EMAIL";
|
||||
email: string;
|
||||
}
|
||||
| {
|
||||
type: "WEBHOOK";
|
||||
url: string;
|
||||
secret?: string;
|
||||
}
|
||||
| {
|
||||
type: "SLACK";
|
||||
channelId: string;
|
||||
channelName: string;
|
||||
integrationId: string | undefined;
|
||||
};
|
||||
};
|
||||
|
||||
export class CreateAlertChannelService extends BaseService {
|
||||
public async call(
|
||||
projectRef: string,
|
||||
userId: string,
|
||||
options: CreateAlertChannelOptions
|
||||
): Promise<ProjectAlertChannel> {
|
||||
const project = await findProjectByRef(projectRef, userId);
|
||||
|
||||
if (!project) {
|
||||
throw new ServiceValidationError("Project not found");
|
||||
}
|
||||
|
||||
// Validate webhook URLs here (not per-route) so every caller is covered.
|
||||
// Delivery re-validates at connect time via safeWebhookFetch.
|
||||
if (options.channel.type === "WEBHOOK") {
|
||||
try {
|
||||
await assertSafeWebhookUrl(options.channel.url);
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafeWebhookUrlError) {
|
||||
throw new ServiceValidationError(error.message);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const environmentTypes =
|
||||
options.environmentTypes.length === 0
|
||||
? (["STAGING", "PRODUCTION"] satisfies RuntimeEnvironmentType[])
|
||||
: options.environmentTypes;
|
||||
|
||||
const existingAlertChannel = options.deduplicationKey
|
||||
? await this._prisma.projectAlertChannel.findFirst({
|
||||
where: {
|
||||
projectId: project.id,
|
||||
deduplicationKey: options.deduplicationKey,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
if (existingAlertChannel) {
|
||||
const updated = await this._prisma.projectAlertChannel.update({
|
||||
where: { id: existingAlertChannel.id },
|
||||
data: {
|
||||
name: options.name,
|
||||
alertTypes: options.alertTypes,
|
||||
type: options.channel.type,
|
||||
properties: await this.#createProperties(options.channel),
|
||||
environmentTypes,
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.alertTypes.includes("ERROR_GROUP")) {
|
||||
await this.#scheduleErrorAlertEvaluation(project.id);
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
const alertChannel = await this._prisma.projectAlertChannel.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("alert_channel"),
|
||||
name: options.name,
|
||||
alertTypes: options.alertTypes,
|
||||
projectId: project.id,
|
||||
type: options.channel.type,
|
||||
properties: await this.#createProperties(options.channel),
|
||||
enabled: true,
|
||||
deduplicationKey: options.deduplicationKey,
|
||||
userProvidedDeduplicationKey: options.deduplicationKey ? true : false,
|
||||
environmentTypes,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.alertTypes.includes("ERROR_GROUP")) {
|
||||
await this.#scheduleErrorAlertEvaluation(project.id);
|
||||
}
|
||||
|
||||
return alertChannel;
|
||||
}
|
||||
|
||||
async #scheduleErrorAlertEvaluation(projectId: string): Promise<void> {
|
||||
await alertsWorker.enqueue({
|
||||
id: `evaluateErrorAlerts:${projectId}`,
|
||||
job: "v3.evaluateErrorAlerts",
|
||||
payload: {
|
||||
projectId,
|
||||
scheduledAt: Date.now(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async #createProperties(channel: CreateAlertChannelOptions["channel"]) {
|
||||
switch (channel.type) {
|
||||
case "EMAIL":
|
||||
return {
|
||||
email: channel.email,
|
||||
};
|
||||
case "WEBHOOK":
|
||||
return {
|
||||
url: channel.url,
|
||||
secret: await encryptSecret(env.ENCRYPTION_KEY, channel.secret ?? nanoid()),
|
||||
version: "v2",
|
||||
};
|
||||
case "SLACK":
|
||||
return {
|
||||
channelId: channel.channelId,
|
||||
channelName: channel.channelName,
|
||||
integrationId: channel.integrationId,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,423 @@
|
||||
import {
|
||||
type ChatPostMessageArguments,
|
||||
ErrorCode,
|
||||
type WebAPIPlatformError,
|
||||
type WebAPIRateLimitedError,
|
||||
} from "@slack/web-api";
|
||||
import { type ProjectAlertChannelType } from "@trigger.dev/database";
|
||||
import assertNever from "assert-never";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { v3ErrorPath } from "~/utils/pathBuilder";
|
||||
import {
|
||||
isIntegrationForService,
|
||||
type OrganizationIntegrationForService,
|
||||
OrgIntegrationRepository,
|
||||
} from "~/models/orgIntegration.server";
|
||||
import {
|
||||
ProjectAlertEmailProperties,
|
||||
ProjectAlertSlackProperties,
|
||||
ProjectAlertWebhookProperties,
|
||||
} from "~/models/projectAlert.server";
|
||||
import { sendAlertEmail } from "~/services/email.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { decryptSecret } from "~/services/secrets/secretStore.server";
|
||||
import { subtle } from "crypto";
|
||||
import { generateErrorGroupWebhookPayload } from "./errorGroupWebhook.server";
|
||||
import { safeWebhookFetch } from "./safeWebhookFetch.server";
|
||||
|
||||
type ErrorAlertClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
interface ErrorAlertPayload {
|
||||
channelId: string;
|
||||
projectId: string;
|
||||
classification: ErrorAlertClassification;
|
||||
error: {
|
||||
fingerprint: string;
|
||||
environmentId: string;
|
||||
environmentSlug: string;
|
||||
environmentName: string;
|
||||
taskIdentifier: string;
|
||||
errorType: string;
|
||||
errorMessage: string;
|
||||
sampleStackTrace: string;
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
occurrenceCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
class SkipRetryError extends Error {}
|
||||
|
||||
export class DeliverErrorGroupAlertService {
|
||||
async call(payload: ErrorAlertPayload): Promise<void> {
|
||||
const channel = await prisma.projectAlertChannel.findFirst({
|
||||
where: { id: payload.channelId, enabled: true },
|
||||
include: {
|
||||
project: {
|
||||
include: {
|
||||
organization: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!channel) {
|
||||
logger.warn("[DeliverErrorGroupAlert] Channel not found or disabled", {
|
||||
channelId: payload.channelId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const errorLink = this.#buildErrorLink(
|
||||
channel.project.organization,
|
||||
channel.project,
|
||||
payload.error
|
||||
);
|
||||
|
||||
try {
|
||||
switch (channel.type) {
|
||||
case "EMAIL":
|
||||
await this.#sendEmail(channel, payload, errorLink);
|
||||
break;
|
||||
case "SLACK":
|
||||
await this.#sendSlack(channel, payload, errorLink);
|
||||
break;
|
||||
case "WEBHOOK":
|
||||
await this.#sendWebhook(channel, payload, errorLink);
|
||||
break;
|
||||
default:
|
||||
assertNever(channel.type);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof SkipRetryError) {
|
||||
logger.warn("[DeliverErrorGroupAlert] Skipping retry", {
|
||||
reason: (error as Error).message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
#buildErrorLink(
|
||||
organization: { slug: string },
|
||||
project: { slug: string },
|
||||
error: ErrorAlertPayload["error"]
|
||||
): string {
|
||||
return `${env.APP_ORIGIN}${v3ErrorPath(organization, project, { slug: error.environmentSlug }, { fingerprint: error.fingerprint })}`;
|
||||
}
|
||||
|
||||
#classificationLabel(classification: ErrorAlertClassification): string {
|
||||
switch (classification) {
|
||||
case "new_issue":
|
||||
return "New error";
|
||||
case "regression":
|
||||
return "Regression";
|
||||
case "unignored":
|
||||
return "Error resurfaced";
|
||||
}
|
||||
}
|
||||
|
||||
async #sendEmail(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: { name: string; organization: { title: string } };
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties);
|
||||
if (!emailProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse email properties", {
|
||||
issues: emailProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await sendAlertEmail({
|
||||
email: "alert-error-group",
|
||||
to: emailProperties.data.email,
|
||||
classification: payload.classification,
|
||||
taskIdentifier: payload.error.taskIdentifier,
|
||||
environment: payload.error.environmentName,
|
||||
error: {
|
||||
message: payload.error.errorMessage,
|
||||
type: payload.error.errorType,
|
||||
stackTrace: payload.error.sampleStackTrace || undefined,
|
||||
},
|
||||
occurrenceCount: payload.error.occurrenceCount,
|
||||
errorLink,
|
||||
organization: channel.project.organization.title,
|
||||
project: channel.project.name,
|
||||
});
|
||||
}
|
||||
|
||||
async #sendSlack(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: { organizationId: string; name: string; organization: { title: string } };
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties);
|
||||
if (!slackProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse slack properties", {
|
||||
issues: slackProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const integration = slackProperties.data.integrationId
|
||||
? await prisma.organizationIntegration.findFirst({
|
||||
where: {
|
||||
id: slackProperties.data.integrationId,
|
||||
organizationId: channel.project.organizationId,
|
||||
},
|
||||
include: { tokenReference: true },
|
||||
})
|
||||
: await prisma.organizationIntegration.findFirst({
|
||||
where: {
|
||||
service: "SLACK",
|
||||
organizationId: channel.project.organizationId,
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: { tokenReference: true },
|
||||
});
|
||||
|
||||
if (!integration || !isIntegrationForService(integration, "SLACK")) {
|
||||
logger.error("[DeliverErrorGroupAlert] Slack integration not found");
|
||||
return;
|
||||
}
|
||||
|
||||
const message = this.#buildErrorGroupSlackMessage(payload, errorLink, channel.project.name);
|
||||
|
||||
await this.#postSlackMessage(integration, {
|
||||
channel: slackProperties.data.channelId,
|
||||
...message,
|
||||
} as ChatPostMessageArguments);
|
||||
}
|
||||
|
||||
async #sendWebhook(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: {
|
||||
id: string;
|
||||
externalRef: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
organizationId: string;
|
||||
organization: { slug: string; title: string };
|
||||
};
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties);
|
||||
if (!webhookProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse webhook properties", {
|
||||
issues: webhookProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const webhookPayload = generateErrorGroupWebhookPayload({
|
||||
classification: payload.classification,
|
||||
error: payload.error,
|
||||
organization: {
|
||||
id: channel.project.organizationId,
|
||||
slug: channel.project.organization.slug,
|
||||
name: channel.project.organization.title,
|
||||
},
|
||||
project: {
|
||||
id: channel.project.id,
|
||||
externalRef: channel.project.externalRef,
|
||||
slug: channel.project.slug,
|
||||
name: channel.project.name,
|
||||
},
|
||||
dashboardUrl: errorLink,
|
||||
});
|
||||
|
||||
const rawPayload = JSON.stringify(webhookPayload);
|
||||
const hashPayload = Buffer.from(rawPayload, "utf-8");
|
||||
const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret);
|
||||
const hmacSecret = Buffer.from(secret, "utf-8");
|
||||
const key = await subtle.importKey(
|
||||
"raw",
|
||||
hmacSecret,
|
||||
{ name: "HMAC", hash: "SHA-256" },
|
||||
false,
|
||||
["sign"]
|
||||
);
|
||||
const signature = await subtle.sign("HMAC", key, hashPayload);
|
||||
const signatureHex = Buffer.from(signature).toString("hex");
|
||||
|
||||
// Deliver via the SSRF-safe wrapper (see safeWebhookFetch.server.ts).
|
||||
const response = await safeWebhookFetch(webhookProperties.data.url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-trigger-signature-hmacsha256": signatureHex,
|
||||
},
|
||||
body: rawPayload,
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.info("[DeliverErrorGroupAlert] Failed to send webhook", {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
url: webhookProperties.data.url,
|
||||
});
|
||||
throw new Error(`Failed to send error group alert webhook to ${webhookProperties.data.url}`);
|
||||
}
|
||||
}
|
||||
|
||||
async #postSlackMessage(
|
||||
integration: OrganizationIntegrationForService<"SLACK">,
|
||||
message: ChatPostMessageArguments
|
||||
) {
|
||||
const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration(
|
||||
integration,
|
||||
{ forceBotToken: true }
|
||||
);
|
||||
|
||||
try {
|
||||
return await client.chat.postMessage({
|
||||
...message,
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isWebAPIRateLimitedError(error)) {
|
||||
throw new Error("Slack rate limited");
|
||||
}
|
||||
if (isWebAPIPlatformError(error)) {
|
||||
if (
|
||||
(error as WebAPIPlatformError).data.error === "invalid_blocks" ||
|
||||
(error as WebAPIPlatformError).data.error === "account_inactive"
|
||||
) {
|
||||
throw new SkipRetryError(`Slack: ${(error as WebAPIPlatformError).data.error}`);
|
||||
}
|
||||
throw new Error("Slack platform error");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
#buildErrorGroupSlackMessage(
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string,
|
||||
projectName: string
|
||||
): { text: string; blocks: object[]; attachments: object[] } {
|
||||
const label = this.#classificationLabel(payload.classification);
|
||||
const errorType = payload.error.errorType || "Error";
|
||||
const task = payload.error.taskIdentifier;
|
||||
const envName = payload.error.environmentName;
|
||||
|
||||
return {
|
||||
text: `${label}: ${errorType} in ${task} [${envName}]`,
|
||||
blocks: [
|
||||
{
|
||||
type: "section",
|
||||
text: {
|
||||
type: "mrkdwn",
|
||||
text: `*${label} in ${task} [${envName}]*`,
|
||||
},
|
||||
},
|
||||
],
|
||||
attachments: [
|
||||
{
|
||||
color: "danger",
|
||||
blocks: [
|
||||
{
|
||||
type: "section",
|
||||
text: {
|
||||
type: "mrkdwn",
|
||||
text: this.#wrapInCodeBlock(
|
||||
payload.error.sampleStackTrace || payload.error.errorMessage
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "section",
|
||||
fields: [
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Task:*\n${task}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Environment:*\n${envName}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Project:*\n${projectName}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Occurrences:*\n${payload.error.occurrenceCount}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Last seen:*\n${this.#formatTimestamp(new Date(Number(payload.error.lastSeen)))}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "actions",
|
||||
elements: [
|
||||
{
|
||||
type: "button",
|
||||
text: { type: "plain_text", text: "Investigate" },
|
||||
url: errorLink,
|
||||
style: "primary",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
#wrapInCodeBlock(text: string, maxLength = 3000) {
|
||||
const wrapperLength = 6; // ``` prefix + ``` suffix
|
||||
const truncationSuffix = "\n\n...truncated — check dashboard for full error";
|
||||
const innerMax = maxLength - wrapperLength;
|
||||
|
||||
const truncated =
|
||||
text.length > innerMax
|
||||
? text.slice(0, innerMax - truncationSuffix.length) + truncationSuffix
|
||||
: text;
|
||||
return `\`\`\`${truncated}\`\`\``;
|
||||
}
|
||||
|
||||
#formatTimestamp(date: Date): string {
|
||||
const unix = Math.floor(date.getTime() / 1000);
|
||||
const fallback =
|
||||
new Intl.DateTimeFormat("en-US", {
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
year: "numeric",
|
||||
hour: "numeric",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: true,
|
||||
timeZone: "UTC",
|
||||
}).format(date) + " UTC";
|
||||
return `<!date^${unix}^{date_short_pretty} {time_secs}|${fallback}>`;
|
||||
}
|
||||
}
|
||||
|
||||
function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError {
|
||||
return (error as WebAPIPlatformError).code === ErrorCode.PlatformError;
|
||||
}
|
||||
|
||||
function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError {
|
||||
return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError;
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
import { type ActiveErrorsSinceQueryResult } from "@internal/clickhouse";
|
||||
import {
|
||||
type ErrorGroupState,
|
||||
type PrismaClientOrTransaction,
|
||||
type ProjectAlertChannel,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import { $replica, prisma } from "~/db.server";
|
||||
import { ErrorAlertConfig } from "~/models/projectAlert.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
|
||||
type ErrorClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
interface AlertableError {
|
||||
classification: ErrorClassification;
|
||||
error: ActiveErrorsSinceQueryResult;
|
||||
environmentSlug: string;
|
||||
environmentName: string;
|
||||
}
|
||||
|
||||
interface ResolvedEnvironment {
|
||||
id: string;
|
||||
slug: string;
|
||||
type: RuntimeEnvironmentType;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
const DEFAULT_INTERVAL_MS = 300_000;
|
||||
|
||||
/**
|
||||
* For a project evalutes whether to send error alerts
|
||||
*
|
||||
* Alerts are sent if an error is
|
||||
* 1. A new issue
|
||||
* 2. A regression (was resolved and now back)
|
||||
* 3. Unignored (was ignored and is no longer)
|
||||
*
|
||||
* Unignored happens in 3 situations
|
||||
* 1. It was ignored with a future date, and that's now in the past
|
||||
* 2. It was ignored until reaching an error rate (e.g. 10/minute) and that has been exceeded
|
||||
* 3. It was ignored until reaching a total occurrence count (e.g. 1,000) and that has been exceeded
|
||||
*/
|
||||
export class ErrorAlertEvaluator {
|
||||
constructor(
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
protected readonly _replica: PrismaClientOrTransaction = $replica
|
||||
) {}
|
||||
|
||||
async evaluate(projectId: string, scheduledAt: number): Promise<void> {
|
||||
const nextScheduledAt = Date.now();
|
||||
|
||||
const channels = await this.resolveChannels(projectId);
|
||||
if (channels.length === 0) {
|
||||
logger.info("[ErrorAlertEvaluator] No active ERROR_GROUP channels, self-terminating", {
|
||||
projectId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const minIntervalMs = this.computeMinInterval(channels);
|
||||
const windowMs = nextScheduledAt - scheduledAt;
|
||||
|
||||
if (windowMs > minIntervalMs * 2) {
|
||||
logger.info("[ErrorAlertEvaluator] Large evaluation window (gap detected)", {
|
||||
projectId,
|
||||
scheduledAt,
|
||||
nextScheduledAt,
|
||||
windowMs,
|
||||
minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
const allEnvTypes = this.collectEnvironmentTypes(channels);
|
||||
|
||||
try {
|
||||
const [project, environments] = await Promise.all([
|
||||
this._replica.project.findFirst({
|
||||
where: { id: projectId },
|
||||
select: { organizationId: true },
|
||||
}),
|
||||
this.resolveEnvironments(projectId, allEnvTypes),
|
||||
]);
|
||||
|
||||
if (!project) {
|
||||
logger.error("[ErrorAlertEvaluator] Project not found", { projectId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (environments.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const envIds = environments.map((e) => e.id);
|
||||
const envMap = new Map(environments.map((e) => [e.id, e]));
|
||||
const channelsByEnvId = this.buildChannelsByEnvId(channels, environments);
|
||||
|
||||
const activeErrors = await this.getActiveErrors(
|
||||
project.organizationId,
|
||||
projectId,
|
||||
envIds,
|
||||
scheduledAt
|
||||
);
|
||||
|
||||
if (activeErrors.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const states = await this.getErrorGroupStates(activeErrors);
|
||||
const stateMap = this.buildStateMap(states);
|
||||
|
||||
const occurrenceCounts = await this.getOccurrenceCountsSince(
|
||||
project.organizationId,
|
||||
projectId,
|
||||
envIds,
|
||||
scheduledAt
|
||||
);
|
||||
const occurrenceMap = this.buildOccurrenceMap(occurrenceCounts);
|
||||
|
||||
const alertableErrors: AlertableError[] = [];
|
||||
|
||||
for (const error of activeErrors) {
|
||||
const key = `${error.environment_id}:${error.task_identifier}:${error.error_fingerprint}`;
|
||||
const state = stateMap.get(key);
|
||||
const env = envMap.get(error.environment_id);
|
||||
const firstSeenMs = Number(error.first_seen);
|
||||
|
||||
const classification = this.classifyError(error, state, firstSeenMs, scheduledAt, {
|
||||
occurrencesSince: occurrenceMap.get(key) ?? 0,
|
||||
windowMs,
|
||||
totalOccurrenceCount: error.occurrence_count,
|
||||
});
|
||||
|
||||
if (classification) {
|
||||
alertableErrors.push({
|
||||
classification,
|
||||
error,
|
||||
environmentSlug: env?.slug ?? "",
|
||||
environmentName: env?.displayName ?? error.environment_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const alertable of alertableErrors) {
|
||||
const envChannels = channelsByEnvId.get(alertable.error.environment_id) ?? [];
|
||||
for (const channel of envChannels) {
|
||||
await alertsWorker.enqueue({
|
||||
id: `deliverErrorGroupAlert:${channel.id}:${alertable.error.error_fingerprint}:${scheduledAt}`,
|
||||
job: "v3.deliverErrorGroupAlert",
|
||||
payload: {
|
||||
channelId: channel.id,
|
||||
projectId,
|
||||
classification: alertable.classification,
|
||||
error: {
|
||||
fingerprint: alertable.error.error_fingerprint,
|
||||
environmentId: alertable.error.environment_id,
|
||||
environmentSlug: alertable.environmentSlug,
|
||||
environmentName: alertable.environmentName,
|
||||
taskIdentifier: alertable.error.task_identifier,
|
||||
errorType: alertable.error.error_type,
|
||||
errorMessage: alertable.error.error_message,
|
||||
sampleStackTrace: alertable.error.sample_stack_trace,
|
||||
firstSeen: alertable.error.first_seen,
|
||||
lastSeen: alertable.error.last_seen,
|
||||
occurrenceCount: alertable.error.occurrence_count,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateErrorGroupStates(
|
||||
alertableErrors,
|
||||
stateMap,
|
||||
project.organizationId,
|
||||
projectId
|
||||
);
|
||||
|
||||
logger.info("[ErrorAlertEvaluator] Evaluation complete", {
|
||||
projectId,
|
||||
activeErrors: activeErrors.length,
|
||||
alertableErrors: alertableErrors.length,
|
||||
deliveryJobsEnqueued: alertableErrors.reduce(
|
||||
(sum, a) => sum + (channelsByEnvId.get(a.error.environment_id)?.length ?? 0),
|
||||
0
|
||||
),
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("[ErrorAlertEvaluator] Evaluation failed, will retry on next cycle", {
|
||||
projectId,
|
||||
error,
|
||||
});
|
||||
} finally {
|
||||
await this.selfChain(projectId, nextScheduledAt, minIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private classifyError(
|
||||
error: ActiveErrorsSinceQueryResult,
|
||||
state: ErrorGroupState | undefined,
|
||||
firstSeenMs: number,
|
||||
scheduledAt: number,
|
||||
thresholdContext: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
|
||||
): ErrorClassification | null {
|
||||
if (!state) {
|
||||
return firstSeenMs > scheduledAt ? "new_issue" : null;
|
||||
}
|
||||
|
||||
switch (state.status) {
|
||||
case "UNRESOLVED":
|
||||
return null;
|
||||
|
||||
case "RESOLVED": {
|
||||
if (!state.resolvedAt) return null;
|
||||
const lastSeenMs = Number(error.last_seen);
|
||||
return lastSeenMs > state.resolvedAt.getTime() ? "regression" : null;
|
||||
}
|
||||
|
||||
case "IGNORED":
|
||||
return this.isIgnoreBreached(state, thresholdContext) ? "unignored" : null;
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isIgnoreBreached(
|
||||
state: ErrorGroupState,
|
||||
context: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
|
||||
): boolean {
|
||||
if (state.ignoredUntil && state.ignoredUntil.getTime() <= Date.now()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
state.ignoredUntilOccurrenceRate !== null &&
|
||||
state.ignoredUntilOccurrenceRate !== undefined
|
||||
) {
|
||||
const windowMinutes = Math.max(context.windowMs / 60_000, 1);
|
||||
const rate = context.occurrencesSince / windowMinutes;
|
||||
if (rate > state.ignoredUntilOccurrenceRate) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (state.ignoredUntilTotalOccurrences != null && state.ignoredAtOccurrenceCount != null) {
|
||||
const occurrencesSinceIgnored =
|
||||
context.totalOccurrenceCount - Number(state.ignoredAtOccurrenceCount);
|
||||
if (occurrencesSinceIgnored >= state.ignoredUntilTotalOccurrences) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async resolveChannels(projectId: string): Promise<ProjectAlertChannel[]> {
|
||||
return this._replica.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
alertTypes: { has: "ERROR_GROUP" },
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private computeMinInterval(channels: ProjectAlertChannel[]): number {
|
||||
let min = DEFAULT_INTERVAL_MS;
|
||||
for (const ch of channels) {
|
||||
const config = ErrorAlertConfig.safeParse(ch.errorAlertConfig);
|
||||
if (config.success) {
|
||||
min = Math.min(min, config.data.evaluationIntervalMs);
|
||||
}
|
||||
}
|
||||
return min;
|
||||
}
|
||||
|
||||
private collectEnvironmentTypes(channels: ProjectAlertChannel[]): RuntimeEnvironmentType[] {
|
||||
const types = new Set<RuntimeEnvironmentType>();
|
||||
for (const ch of channels) {
|
||||
for (const t of ch.environmentTypes) {
|
||||
types.add(t);
|
||||
}
|
||||
}
|
||||
return Array.from(types);
|
||||
}
|
||||
|
||||
private async resolveEnvironments(
|
||||
projectId: string,
|
||||
types: RuntimeEnvironmentType[]
|
||||
): Promise<ResolvedEnvironment[]> {
|
||||
const envs = await this._replica.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
type: { in: types },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
slug: true,
|
||||
branchName: true,
|
||||
},
|
||||
});
|
||||
|
||||
return envs.map((e) => ({
|
||||
id: e.id,
|
||||
slug: e.slug,
|
||||
type: e.type,
|
||||
displayName: e.branchName ?? e.slug,
|
||||
}));
|
||||
}
|
||||
|
||||
private buildChannelsByEnvId(
|
||||
channels: ProjectAlertChannel[],
|
||||
environments: ResolvedEnvironment[]
|
||||
): Map<string, ProjectAlertChannel[]> {
|
||||
const result = new Map<string, ProjectAlertChannel[]>();
|
||||
for (const env of environments) {
|
||||
const matching = channels.filter((ch) => ch.environmentTypes.includes(env.type));
|
||||
if (matching.length > 0) {
|
||||
result.set(env.id, matching);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async getActiveErrors(
|
||||
organizationId: string,
|
||||
projectId: string,
|
||||
envIds: string[],
|
||||
scheduledAt: number
|
||||
): Promise<ActiveErrorsSinceQueryResult[]> {
|
||||
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"query"
|
||||
);
|
||||
const qb = queryClickhouse.errors.activeErrorsSinceQueryBuilder();
|
||||
qb.where("organization_id = {organizationId: String}", { organizationId });
|
||||
qb.where("project_id = {projectId: String}", { projectId });
|
||||
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
|
||||
qb.groupBy("environment_id, task_identifier, error_fingerprint");
|
||||
qb.having("toInt64(last_seen) > {scheduledAt: Int64}", {
|
||||
scheduledAt,
|
||||
});
|
||||
|
||||
const [err, results] = await qb.execute();
|
||||
if (err) {
|
||||
logger.error("[ErrorAlertEvaluator] Failed to query active errors", { error: err });
|
||||
return [];
|
||||
}
|
||||
return results ?? [];
|
||||
}
|
||||
|
||||
private async getErrorGroupStates(
|
||||
activeErrors: ActiveErrorsSinceQueryResult[]
|
||||
): Promise<ErrorGroupState[]> {
|
||||
if (activeErrors.length === 0) return [];
|
||||
|
||||
return this._replica.errorGroupState.findMany({
|
||||
where: {
|
||||
OR: activeErrors.map((e) => ({
|
||||
environmentId: e.environment_id,
|
||||
taskIdentifier: e.task_identifier,
|
||||
errorFingerprint: e.error_fingerprint,
|
||||
})),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private buildStateMap(states: ErrorGroupState[]): Map<string, ErrorGroupState> {
|
||||
const map = new Map<string, ErrorGroupState>();
|
||||
for (const s of states) {
|
||||
map.set(`${s.environmentId}:${s.taskIdentifier}:${s.errorFingerprint}`, s);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private async getOccurrenceCountsSince(
|
||||
organizationId: string,
|
||||
projectId: string,
|
||||
envIds: string[],
|
||||
scheduledAt: number
|
||||
): Promise<
|
||||
Array<{
|
||||
environment_id: string;
|
||||
task_identifier: string;
|
||||
error_fingerprint: string;
|
||||
occurrences_since: number;
|
||||
}>
|
||||
> {
|
||||
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"query"
|
||||
);
|
||||
const qb = queryClickhouse.errors.occurrenceCountsSinceQueryBuilder();
|
||||
qb.where("organization_id = {organizationId: String}", { organizationId });
|
||||
qb.where("project_id = {projectId: String}", { projectId });
|
||||
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
|
||||
qb.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({scheduledAt: Int64}))", {
|
||||
scheduledAt,
|
||||
});
|
||||
qb.groupBy("environment_id, task_identifier, error_fingerprint");
|
||||
|
||||
const [err, results] = await qb.execute();
|
||||
if (err) {
|
||||
logger.error("[ErrorAlertEvaluator] Failed to query occurrence counts", { error: err });
|
||||
return [];
|
||||
}
|
||||
return results ?? [];
|
||||
}
|
||||
|
||||
private buildOccurrenceMap(
|
||||
counts: Array<{
|
||||
environment_id: string;
|
||||
task_identifier: string;
|
||||
error_fingerprint: string;
|
||||
occurrences_since: number;
|
||||
}>
|
||||
): Map<string, number> {
|
||||
const map = new Map<string, number>();
|
||||
for (const c of counts) {
|
||||
map.set(
|
||||
`${c.environment_id}:${c.task_identifier}:${c.error_fingerprint}`,
|
||||
c.occurrences_since
|
||||
);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private async updateErrorGroupStates(
|
||||
alertableErrors: AlertableError[],
|
||||
stateMap: Map<string, ErrorGroupState>,
|
||||
organizationId: string,
|
||||
projectId: string
|
||||
): Promise<void> {
|
||||
for (const alertable of alertableErrors) {
|
||||
const key = `${alertable.error.environment_id}:${alertable.error.task_identifier}:${alertable.error.error_fingerprint}`;
|
||||
const state = stateMap.get(key);
|
||||
|
||||
if (state) {
|
||||
await this._prisma.errorGroupState.update({
|
||||
where: { id: state.id },
|
||||
data: {
|
||||
status: "UNRESOLVED",
|
||||
ignoredUntil: null,
|
||||
ignoredUntilOccurrenceRate: null,
|
||||
ignoredUntilTotalOccurrences: null,
|
||||
ignoredAtOccurrenceCount: null,
|
||||
ignoredAt: null,
|
||||
ignoredReason: null,
|
||||
ignoredByUserId: null,
|
||||
resolvedAt: null,
|
||||
resolvedInVersion: null,
|
||||
resolvedBy: null,
|
||||
},
|
||||
});
|
||||
} else if (alertable.classification === "new_issue") {
|
||||
await this._prisma.errorGroupState.create({
|
||||
data: {
|
||||
organizationId,
|
||||
projectId,
|
||||
environmentId: alertable.error.environment_id,
|
||||
taskIdentifier: alertable.error.task_identifier,
|
||||
errorFingerprint: alertable.error.error_fingerprint,
|
||||
status: "UNRESOLVED",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async selfChain(
|
||||
projectId: string,
|
||||
nextScheduledAt: number,
|
||||
intervalMs: number
|
||||
): Promise<void> {
|
||||
await alertsWorker.enqueue({
|
||||
id: `evaluateErrorAlerts:${projectId}`,
|
||||
job: "v3.evaluateErrorAlerts",
|
||||
payload: {
|
||||
projectId,
|
||||
scheduledAt: nextScheduledAt,
|
||||
},
|
||||
availableAt: new Date(nextScheduledAt + intervalMs),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import type { ErrorWebhook } from "@trigger.dev/core/v3/schemas";
|
||||
|
||||
export type ErrorAlertClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
export type ErrorGroupAlertData = {
|
||||
classification: ErrorAlertClassification;
|
||||
error: {
|
||||
fingerprint: string;
|
||||
environmentId: string;
|
||||
environmentName: string;
|
||||
taskIdentifier: string;
|
||||
errorType: string;
|
||||
errorMessage: string;
|
||||
sampleStackTrace: string;
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
occurrenceCount: number;
|
||||
};
|
||||
organization: {
|
||||
id: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
};
|
||||
project: {
|
||||
id: string;
|
||||
externalRef: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
};
|
||||
dashboardUrl: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generates a webhook payload for an error group alert that conforms to the
|
||||
* ErrorWebhook schema from @trigger.dev/core/v3/schemas
|
||||
*/
|
||||
export function generateErrorGroupWebhookPayload(data: ErrorGroupAlertData): ErrorWebhook {
|
||||
return {
|
||||
id: nanoid(),
|
||||
created: new Date(),
|
||||
webhookVersion: "2025-01-01",
|
||||
type: "alert.error" as const,
|
||||
object: {
|
||||
classification: data.classification,
|
||||
error: {
|
||||
fingerprint: data.error.fingerprint,
|
||||
type: data.error.errorType,
|
||||
message: data.error.errorMessage,
|
||||
stackTrace: data.error.sampleStackTrace || undefined,
|
||||
firstSeen: new Date(Number(data.error.firstSeen)),
|
||||
lastSeen: new Date(Number(data.error.lastSeen)),
|
||||
occurrenceCount: data.error.occurrenceCount,
|
||||
taskIdentifier: data.error.taskIdentifier,
|
||||
},
|
||||
environment: {
|
||||
id: data.error.environmentId,
|
||||
name: data.error.environmentName,
|
||||
},
|
||||
organization: {
|
||||
id: data.organization.id,
|
||||
slug: data.organization.slug,
|
||||
name: data.organization.name,
|
||||
},
|
||||
project: {
|
||||
id: data.project.id,
|
||||
ref: data.project.externalRef,
|
||||
slug: data.project.slug,
|
||||
name: data.project.name,
|
||||
},
|
||||
dashboardUrl: data.dashboardUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import type {
|
||||
ProjectAlertChannel,
|
||||
ProjectAlertType,
|
||||
WorkerDeployment,
|
||||
} from "@trigger.dev/database";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { DeliverAlertService } from "./deliverAlert.server";
|
||||
|
||||
export class PerformDeploymentAlertsService extends BaseService {
|
||||
public async call(deploymentId: string) {
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: { id: deploymentId },
|
||||
include: {
|
||||
environment: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
return;
|
||||
}
|
||||
|
||||
const alertType =
|
||||
deployment.status === "DEPLOYED" ? "DEPLOYMENT_SUCCESS" : "DEPLOYMENT_FAILURE";
|
||||
|
||||
// Find all the alert channels
|
||||
const alertChannels = await this._prisma.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId: deployment.projectId,
|
||||
alertTypes: {
|
||||
has: alertType,
|
||||
},
|
||||
environmentTypes: {
|
||||
has: deployment.environment.type,
|
||||
},
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const alertChannel of alertChannels) {
|
||||
await this.#createAndSendAlert(alertChannel, deployment, alertType);
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndSendAlert(
|
||||
alertChannel: ProjectAlertChannel,
|
||||
deployment: WorkerDeployment,
|
||||
alertType: ProjectAlertType
|
||||
) {
|
||||
await DeliverAlertService.createAndSendAlert(
|
||||
{
|
||||
channelId: alertChannel.id,
|
||||
channelType: alertChannel.type,
|
||||
projectId: deployment.projectId,
|
||||
environmentId: deployment.environmentId,
|
||||
alertType,
|
||||
deploymentId: deployment.id,
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
}
|
||||
|
||||
static async enqueue(deploymentId: string, runAt?: Date) {
|
||||
return await alertsWorker.enqueue({
|
||||
id: `performDeploymentAlerts:${deploymentId}`,
|
||||
job: "v3.performDeploymentAlerts",
|
||||
payload: { deploymentId },
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { type RunStore } from "@internal/run-store";
|
||||
import { type Prisma, type ProjectAlertChannel } from "@trigger.dev/database";
|
||||
import { type PrismaClientOrTransaction, type prisma } from "~/db.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import type { ControlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { controlPlaneResolver as defaultControlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { DeliverAlertService } from "./deliverAlert.server";
|
||||
|
||||
// The alert hydration reads only run-ops scalars (id/projectId/runtimeEnvironmentId); the env's
|
||||
// type (and its parent's) is resolved via the control-plane resolver so the run-ops DB can split
|
||||
// without a cross-provider join. The prior `lockedBy` + `runtimeEnvironment` includes were unused.
|
||||
type FoundRun = Prisma.Result<
|
||||
typeof prisma.taskRun,
|
||||
{ select: { id: true; projectId: true; runtimeEnvironmentId: true } },
|
||||
"findUniqueOrThrow"
|
||||
>;
|
||||
|
||||
export class PerformTaskRunAlertsService extends BaseService {
|
||||
#controlPlaneResolver: ControlPlaneResolver;
|
||||
|
||||
constructor(
|
||||
opts: {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
replica?: PrismaClientOrTransaction;
|
||||
runStore?: RunStore;
|
||||
controlPlaneResolver?: ControlPlaneResolver;
|
||||
} = {}
|
||||
) {
|
||||
super(opts.prisma, opts.replica, opts.runStore);
|
||||
this.#controlPlaneResolver = opts.controlPlaneResolver ?? defaultControlPlaneResolver;
|
||||
}
|
||||
|
||||
public async call(runId: string) {
|
||||
const run = await this.runStore.findRun(
|
||||
{ id: runId },
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
projectId: true,
|
||||
runtimeEnvironmentId: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
return;
|
||||
}
|
||||
|
||||
const env = await this.#controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId);
|
||||
|
||||
if (!env) {
|
||||
return;
|
||||
}
|
||||
|
||||
const alertChannels = await this._prisma.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId: run.projectId,
|
||||
alertTypes: {
|
||||
has: "TASK_RUN",
|
||||
},
|
||||
environmentTypes: {
|
||||
has: env.parentEnvironmentType ?? env.type,
|
||||
},
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const alertChannel of alertChannels) {
|
||||
await this.#createAndSendAlert(alertChannel, run);
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndSendAlert(alertChannel: ProjectAlertChannel, run: FoundRun) {
|
||||
await DeliverAlertService.createAndSendAlert(
|
||||
{
|
||||
channelId: alertChannel.id,
|
||||
channelType: alertChannel.type,
|
||||
projectId: run.projectId,
|
||||
environmentId: run.runtimeEnvironmentId,
|
||||
alertType: "TASK_RUN",
|
||||
taskRunId: run.id,
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
}
|
||||
|
||||
static async enqueue(runId: string, runAt?: Date) {
|
||||
return await alertsWorker.enqueue({
|
||||
id: `performTaskRunAlerts:${runId}`,
|
||||
job: "v3.performTaskRunAlerts",
|
||||
payload: { runId },
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { promises as dnsPromises } from "node:dns";
|
||||
import type { LookupFunction } from "node:net";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import {
|
||||
assertAddressAllowed,
|
||||
assertSafeWebhookUrl,
|
||||
assertSafeWebhookUrlLexical,
|
||||
UnsafeWebhookUrlError,
|
||||
} from "./safeWebhookUrl.server";
|
||||
|
||||
/**
|
||||
* `fetch`-like wrapper for delivering user-supplied webhook URLs. The lexical
|
||||
* check is shared with the storage-time gate (`assertSafeWebhookUrlLexical`).
|
||||
*
|
||||
* Validation is bound to the actual connection: the request goes through
|
||||
* `node:http`/`node:https` with a custom DNS `lookup` that validates every
|
||||
* resolved address before the socket connects, so the connected address is the
|
||||
* one that was checked. Redirects are followed manually and re-validated per
|
||||
* hop, capped at `MAX_REDIRECTS`.
|
||||
*/
|
||||
|
||||
// Re-exported so callers/tests don't reach into the underlying module.
|
||||
export { assertSafeWebhookUrl, assertSafeWebhookUrlLexical, UnsafeWebhookUrlError };
|
||||
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
export type SafeWebhookFetchInit = {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: string | Buffer;
|
||||
signal?: AbortSignal;
|
||||
redirectLimit?: number;
|
||||
};
|
||||
|
||||
// DNS lookup that validates every resolved address before handing it to the
|
||||
// connector; any unsafe address fails the whole lookup. On error we pass an
|
||||
// empty address list, which net ignores when err is set.
|
||||
const safeLookup: LookupFunction = (hostname, options, callback) => {
|
||||
dnsPromises
|
||||
.lookup(hostname, {
|
||||
all: true,
|
||||
family: options.family,
|
||||
hints: options.hints,
|
||||
verbatim: options.verbatim,
|
||||
})
|
||||
.then((addresses) => {
|
||||
try {
|
||||
for (const { address, family } of addresses) {
|
||||
assertAddressAllowed(address, family);
|
||||
}
|
||||
} catch (err) {
|
||||
callback(err as NodeJS.ErrnoException, []);
|
||||
return;
|
||||
}
|
||||
if (options.all) {
|
||||
callback(null, addresses);
|
||||
} else {
|
||||
callback(null, addresses[0].address, addresses[0].family);
|
||||
}
|
||||
})
|
||||
.catch((err) => callback(err as NodeJS.ErrnoException, []));
|
||||
};
|
||||
|
||||
// Single request with no redirect following, using the validating lookup. The
|
||||
// response body is drained and discarded (callers only need status / headers),
|
||||
// which also frees the socket.
|
||||
function requestOnce(urlStr: string, init: SafeWebhookFetchInit): Promise<Response> {
|
||||
const url = new URL(urlStr);
|
||||
const mod = url.protocol === "https:" ? https : http;
|
||||
// Set Content-Length explicitly (as fetch does for string/Buffer bodies)
|
||||
// rather than falling back to chunked transfer-encoding, which some
|
||||
// webhook receivers reject.
|
||||
const headers: Record<string, string> = { ...(init.headers ?? {}) };
|
||||
if (
|
||||
init.body != null &&
|
||||
headers["content-length"] === undefined &&
|
||||
headers["Content-Length"] === undefined
|
||||
) {
|
||||
headers["content-length"] = String(Buffer.byteLength(init.body));
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = mod.request(
|
||||
url,
|
||||
{
|
||||
method: init.method ?? "GET",
|
||||
headers,
|
||||
lookup: safeLookup,
|
||||
signal: init.signal,
|
||||
},
|
||||
(res) => {
|
||||
res.on("data", () => {});
|
||||
res.on("end", () => {
|
||||
const responseHeaders = new Headers();
|
||||
for (const [key, value] of Object.entries(res.headers)) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const v of value) responseHeaders.append(key, v);
|
||||
} else if (value !== undefined) {
|
||||
responseHeaders.set(key, value);
|
||||
}
|
||||
}
|
||||
resolve(
|
||||
new Response(null, {
|
||||
status: res.statusCode ?? 502,
|
||||
statusText: res.statusMessage ?? "",
|
||||
headers: responseHeaders,
|
||||
})
|
||||
);
|
||||
});
|
||||
res.on("error", reject);
|
||||
}
|
||||
);
|
||||
req.on("error", reject);
|
||||
if (init.body != null) {
|
||||
req.write(init.body);
|
||||
}
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Tenant-supplied-URL fetch with connection-bound SSRF validation and manual,
|
||||
* per-hop redirect validation.
|
||||
*/
|
||||
export async function safeWebhookFetch(
|
||||
rawUrl: string,
|
||||
init: SafeWebhookFetchInit = {}
|
||||
): Promise<Response> {
|
||||
let nextUrl = assertSafeWebhookUrlLexical(rawUrl).href;
|
||||
const limit = init.redirectLimit ?? MAX_REDIRECTS;
|
||||
|
||||
for (let hop = 0; hop <= limit; hop++) {
|
||||
const response = await requestOnce(nextUrl, init);
|
||||
if (response.status < 300 || response.status >= 400) {
|
||||
return response;
|
||||
}
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return response;
|
||||
if (hop === limit) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Refusing to deliver webhook to ${nextUrl}: exceeded redirect limit (${limit}) following ${location}`
|
||||
);
|
||||
}
|
||||
const target = new URL(location, nextUrl);
|
||||
try {
|
||||
nextUrl = assertSafeWebhookUrlLexical(target.href).href;
|
||||
} catch (err) {
|
||||
logger.warn("Refusing to follow webhook redirect", {
|
||||
from: nextUrl,
|
||||
to: target.href,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// Unreachable — the loop always returns or throws.
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Refusing to deliver webhook to ${nextUrl}: exhausted redirect loop`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Validator for user-supplied webhook URLs that the server fetches later
|
||||
// (alert channels, error-group webhooks). Rejects non-http(s) schemes and
|
||||
// private/loopback/link-local/reserved hosts.
|
||||
//
|
||||
// Two entry points:
|
||||
// - `assertSafeWebhookUrlLexical` — sync, no network. Runs on every
|
||||
// delivery hop; the connect-time bound lookup below is authoritative.
|
||||
// - `assertSafeWebhookUrl` — storage-time gate: lexical check plus a
|
||||
// best-effort DNS resolution for early, friendly rejection.
|
||||
//
|
||||
// The authoritative guard is at delivery time: `safeWebhookFetch` binds
|
||||
// validation into the connection's own DNS lookup, so the address actually
|
||||
// connected to is the one that was checked.
|
||||
|
||||
import { promises as dnsPromises } from "node:dns";
|
||||
|
||||
export class UnsafeWebhookUrlError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "UnsafeWebhookUrlError";
|
||||
}
|
||||
}
|
||||
|
||||
function isUnsafeIPv4(host: string): boolean {
|
||||
// Reject if the host parses as a 4-octet IPv4 in any of the unsafe ranges.
|
||||
const parts = host.split(".");
|
||||
if (parts.length !== 4) return false;
|
||||
const nums = parts.map((p) => Number(p));
|
||||
if (nums.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return false;
|
||||
const [a, b] = nums;
|
||||
// 0.0.0.0/8 (unspecified)
|
||||
if (a === 0) return true;
|
||||
// 127/8 loopback
|
||||
if (a === 127) return true;
|
||||
// 10/8
|
||||
if (a === 10) return true;
|
||||
// 172.16/12
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
// 192.168/16
|
||||
if (a === 192 && b === 168) return true;
|
||||
// 169.254/16 link-local
|
||||
if (a === 169 && b === 254) return true;
|
||||
// 100.64/10 carrier-grade NAT
|
||||
if (a === 100 && b >= 64 && b <= 127) return true;
|
||||
// 224/4 multicast
|
||||
if (a >= 224 && a <= 239) return true;
|
||||
// 240/4 reserved
|
||||
if (a >= 240) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isUnsafeIPv6(host: string): boolean {
|
||||
// URL.hostname keeps the brackets for IPv6 literals ([::1]); DNS results
|
||||
// and IP literals elsewhere are unbracketed. Strip brackets so both work.
|
||||
const lower = (
|
||||
host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host
|
||||
).toLowerCase();
|
||||
// loopback
|
||||
if (lower === "::1") return true;
|
||||
// unspecified
|
||||
if (lower === "::" || lower === "::0" || lower === "0:0:0:0:0:0:0:0") return true;
|
||||
// link-local fe80::/10
|
||||
if (/^fe[89ab][0-9a-f]?:/.test(lower)) return true;
|
||||
// ULA fc00::/7
|
||||
if (/^f[cd][0-9a-f]{2}:/.test(lower)) return true;
|
||||
// multicast ff00::/8
|
||||
if (lower.startsWith("ff")) return true;
|
||||
// IPv4-mapped, dotted form: ::ffff:a.b.c.d
|
||||
const mappedDotted = lower.match(/^::ffff:([0-9.]+)$/);
|
||||
if (mappedDotted && isUnsafeIPv4(mappedDotted[1])) return true;
|
||||
// IPv4-mapped, hex form: ::ffff:7f00:1 (how Node normalizes ::ffff:127.0.0.1).
|
||||
// The two trailing hextets encode the 32-bit IPv4 address.
|
||||
const mappedHex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
||||
if (mappedHex) {
|
||||
const hi = parseInt(mappedHex[1], 16);
|
||||
const lo = parseInt(mappedHex[2], 16);
|
||||
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
||||
if (isUnsafeIPv4(ipv4)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if a resolved IP address falls in a disallowed range. Exposed so the
|
||||
* delivery-time connector can validate the actual address it connects to.
|
||||
*/
|
||||
export function assertAddressAllowed(address: string, family: number): void {
|
||||
if (family === 4 && isUnsafeIPv4(address)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL resolves to a private/loopback/link-local address: ${address}`
|
||||
);
|
||||
}
|
||||
if (family === 6 && isUnsafeIPv6(address)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL resolves to a private/loopback/link-local IPv6 address: ${address}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isUnsafeHostname(host: string): boolean {
|
||||
const lower = host.toLowerCase();
|
||||
if (lower === "localhost" || lower.endsWith(".localhost")) return true;
|
||||
if (lower === "internal" || lower.endsWith(".internal")) return true;
|
||||
if (lower === "local" || lower.endsWith(".local")) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isIPLiteral(host: string): boolean {
|
||||
// Strip brackets: URL.hostname keeps them for IPv6 literals ([::1]).
|
||||
const bare = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
||||
// IPv4: four dot-separated 0-255 octets.
|
||||
if (/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bare)) return true;
|
||||
// IPv6: at least one `:` and only hex / `:` / `.` (the `.` allows
|
||||
// IPv4-mapped notation like ::ffff:1.2.3.4).
|
||||
if (bare.includes(":") && /^[0-9a-fA-F:.]+$/.test(bare)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort storage-time DNS check: resolve `hostname` and throw if a
|
||||
* returned address is unsafe. Not a security boundary — resolution failures
|
||||
* don't block the save, since delivery re-validates at connect time.
|
||||
*/
|
||||
async function assertResolvedAddressesSafe(hostname: string): Promise<void> {
|
||||
let addresses: Array<{ address: string; family: number }>;
|
||||
try {
|
||||
addresses = await dnsPromises.lookup(hostname, { all: true });
|
||||
} catch {
|
||||
// Unresolvable right now — don't block the save; connect-time is authoritative.
|
||||
return;
|
||||
}
|
||||
for (const { address, family } of addresses) {
|
||||
assertAddressAllowed(address, family);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous, no-network SSRF check: scheme allow-list plus IP-literal
|
||||
* and hostname range checks. Used on every delivery hop, where the
|
||||
* connect-time bound lookup is the authoritative range check.
|
||||
*/
|
||||
export function assertSafeWebhookUrlLexical(rawUrl: string): URL {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new UnsafeWebhookUrlError("Webhook URL is not a valid URL");
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new UnsafeWebhookUrlError(`Webhook URL must use http or https (got ${parsed.protocol})`);
|
||||
}
|
||||
const host = parsed.hostname;
|
||||
if (!host) {
|
||||
throw new UnsafeWebhookUrlError("Webhook URL must have a hostname");
|
||||
}
|
||||
if (isUnsafeHostname(host)) {
|
||||
throw new UnsafeWebhookUrlError(`Webhook URL host is not allowed: ${host}`);
|
||||
}
|
||||
if (isUnsafeIPv4(host)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL points at a private/loopback/link-local address: ${host}`
|
||||
);
|
||||
}
|
||||
if (isUnsafeIPv6(host)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL points at a private/loopback/link-local IPv6 address: ${host}`
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage-time gate: lexical check plus a best-effort DNS resolution of
|
||||
* registrable domains, for early rejection before storage.
|
||||
*/
|
||||
export async function assertSafeWebhookUrl(rawUrl: string): Promise<URL> {
|
||||
const parsed = assertSafeWebhookUrlLexical(rawUrl);
|
||||
if (!isIPLiteral(parsed.hostname)) {
|
||||
await assertResolvedAddressesSafe(parsed.hostname);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { ManageConcurrencyPresenter } from "~/presenters/v3/ManageConcurrencyPresenter.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
|
||||
type Input = {
|
||||
userId: string;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
environments: { id: string; amount: number }[];
|
||||
};
|
||||
|
||||
type Result =
|
||||
| {
|
||||
success: true;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export class AllocateConcurrencyService extends BaseService {
|
||||
async call({ userId, projectId, organizationId, environments }: Input): Promise<Result> {
|
||||
// fetch the current concurrency
|
||||
const presenter = new ManageConcurrencyPresenter(this._prisma, this._replica);
|
||||
const [error, result] = await tryCatch(
|
||||
presenter.call({
|
||||
userId,
|
||||
projectId,
|
||||
organizationId,
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Unknown error",
|
||||
};
|
||||
}
|
||||
|
||||
const previousExtra = result.environments.reduce(
|
||||
(acc, e) => Math.max(0, e.maximumConcurrencyLimit - e.planConcurrencyLimit) + acc,
|
||||
0
|
||||
);
|
||||
const requested = new Map(environments.map((e) => [e.id, e.amount]));
|
||||
const newExtra = result.environments.reduce((acc, env) => {
|
||||
const targetExtra = requested.has(env.id)
|
||||
? Math.max(0, requested.get(env.id)!)
|
||||
: Math.max(0, env.maximumConcurrencyLimit - env.planConcurrencyLimit);
|
||||
return acc + targetExtra;
|
||||
}, 0);
|
||||
const change = newExtra - previousExtra;
|
||||
|
||||
const totalExtra = result.extraAllocatedConcurrency + change;
|
||||
|
||||
if (change > result.extraUnallocatedConcurrency) {
|
||||
return {
|
||||
success: false,
|
||||
error: `You don't have enough unallocated concurrency available. You requested ${totalExtra} but only have ${result.extraUnallocatedConcurrency}.`,
|
||||
};
|
||||
}
|
||||
|
||||
for (const environment of environments) {
|
||||
const existingEnvironment = result.environments.find((e) => e.id === environment.id);
|
||||
|
||||
if (!existingEnvironment) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Environment not found ${environment.id}`,
|
||||
};
|
||||
}
|
||||
|
||||
const newConcurrency = existingEnvironment.planConcurrencyLimit + environment.amount;
|
||||
|
||||
const updatedEnvironment = await this._prisma.runtimeEnvironment.update({
|
||||
where: {
|
||||
id: environment.id,
|
||||
},
|
||||
data: {
|
||||
maximumConcurrencyLimit: newConcurrency,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!updatedEnvironment.paused) {
|
||||
await updateEnvConcurrencyLimits(updatedEnvironment);
|
||||
}
|
||||
|
||||
// maximumConcurrencyLimit changed in the control-plane; drop any cached copy.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { env } from "~/env.server";
|
||||
import { createPresignedPost } from "@aws-sdk/s3-presigned-post";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { customAlphabet } from "nanoid";
|
||||
import { errAsync, fromPromise } from "neverthrow";
|
||||
|
||||
const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 24);
|
||||
const objectStoreClient =
|
||||
env.ARTIFACTS_OBJECT_STORE_ACCESS_KEY_ID &&
|
||||
env.ARTIFACTS_OBJECT_STORE_SECRET_ACCESS_KEY &&
|
||||
env.ARTIFACTS_OBJECT_STORE_BASE_URL
|
||||
? new S3Client({
|
||||
credentials: {
|
||||
accessKeyId: env.ARTIFACTS_OBJECT_STORE_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.ARTIFACTS_OBJECT_STORE_SECRET_ACCESS_KEY,
|
||||
},
|
||||
region: env.ARTIFACTS_OBJECT_STORE_REGION,
|
||||
endpoint: env.ARTIFACTS_OBJECT_STORE_BASE_URL,
|
||||
forcePathStyle: true,
|
||||
})
|
||||
: new S3Client();
|
||||
|
||||
const artifactKeyPrefixByType = {
|
||||
deployment_context: "deployments",
|
||||
} as const;
|
||||
const artifactBytesSizeLimitByType = {
|
||||
deployment_context: 100 * 1024 * 1024, // 100MB
|
||||
} as const;
|
||||
|
||||
export class ArtifactsService extends BaseService {
|
||||
private readonly bucket = env.ARTIFACTS_OBJECT_STORE_BUCKET;
|
||||
|
||||
public createArtifact(
|
||||
type: "deployment_context",
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
contentLength?: number
|
||||
) {
|
||||
const limit = artifactBytesSizeLimitByType[type];
|
||||
|
||||
// this is just a validation using client-side data
|
||||
// the actual limit will be enforced by S3
|
||||
if (contentLength && contentLength > limit) {
|
||||
return errAsync({
|
||||
type: "artifact_size_exceeds_limit" as const,
|
||||
contentLength,
|
||||
sizeLimit: limit,
|
||||
});
|
||||
}
|
||||
|
||||
const uniqueId = nanoid();
|
||||
const key = `${artifactKeyPrefixByType[type]}/${authenticatedEnv.project.externalRef}/${authenticatedEnv.slug}/${uniqueId}.tar.gz`;
|
||||
|
||||
return this.createPresignedPost(key, limit, contentLength).map((result) => ({
|
||||
artifactKey: key,
|
||||
uploadUrl: result.url,
|
||||
uploadFields: result.fields,
|
||||
expiresAt: result.expiresAt,
|
||||
}));
|
||||
}
|
||||
|
||||
private createPresignedPost(key: string, sizeLimit: number, contentLength?: number) {
|
||||
if (!this.bucket) {
|
||||
return errAsync({
|
||||
type: "artifacts_bucket_not_configured" as const,
|
||||
});
|
||||
}
|
||||
|
||||
const ttlSeconds = 300; // 5 minutes
|
||||
const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
|
||||
|
||||
return fromPromise(
|
||||
createPresignedPost(objectStoreClient, {
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Conditions: [["content-length-range", 0, sizeLimit]],
|
||||
Fields: {
|
||||
"Content-Type": "application/gzip",
|
||||
},
|
||||
Expires: ttlSeconds,
|
||||
}),
|
||||
(error) => ({
|
||||
type: "failed_to_create_presigned_post" as const,
|
||||
cause: error,
|
||||
})
|
||||
).map((result) => ({
|
||||
...result,
|
||||
expiresAt,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { Span } from "@opentelemetry/api";
|
||||
import { SpanKind } from "@opentelemetry/api";
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { $replica, prisma } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { attributesFromAuthenticatedEnv, tracer } from "../tracer.server";
|
||||
import type { RunEngine } from "../runEngine.server";
|
||||
import { engine } from "../runEngine.server";
|
||||
import { runStore as defaultRunStore } from "../runStore.server";
|
||||
import { ServiceValidationError } from "./common.server";
|
||||
|
||||
export { ServiceValidationError };
|
||||
|
||||
export abstract class BaseService {
|
||||
constructor(
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
protected readonly _replica: PrismaClientOrTransaction = $replica,
|
||||
protected readonly runStore: RunStore = defaultRunStore
|
||||
) {}
|
||||
|
||||
protected async traceWithEnv<T>(
|
||||
trace: string,
|
||||
env: AuthenticatedEnvironment,
|
||||
fn: (span: Span) => Promise<T>
|
||||
): Promise<T> {
|
||||
return tracer.startActiveSpan(
|
||||
`${this.constructor.name}.${trace}`,
|
||||
{ attributes: attributesFromAuthenticatedEnv(env), kind: SpanKind.SERVER },
|
||||
async (span) => {
|
||||
try {
|
||||
return await fn(span);
|
||||
} catch (e) {
|
||||
if (e instanceof ServiceValidationError) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (e instanceof Error) {
|
||||
span.recordException(e);
|
||||
} else {
|
||||
span.recordException(new Error(String(e)));
|
||||
}
|
||||
|
||||
throw e;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export type WithRunEngineOptions<T> = T & {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
engine?: RunEngine;
|
||||
};
|
||||
|
||||
export class WithRunEngine extends BaseService {
|
||||
protected readonly _engine: RunEngine;
|
||||
|
||||
constructor(opts: { prisma?: PrismaClientOrTransaction; engine?: RunEngine } = {}) {
|
||||
super(opts.prisma);
|
||||
this._engine = opts.engine ?? engine;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import { BatchId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
|
||||
/**
|
||||
* Resolve the BatchTaskRun id for `batchId` (accepting either the friendlyId or
|
||||
* the internal id) only if `userId` is a member of the batch's owning
|
||||
* organization. Returns null otherwise. Batch lookup goes through runStore so
|
||||
* batches resident in either run-store database are visible.
|
||||
*/
|
||||
export async function findBatchRunIdForUser(
|
||||
prisma: PrismaClientOrTransaction,
|
||||
store: RunStore,
|
||||
batchId: string,
|
||||
userId: string
|
||||
): Promise<string | null> {
|
||||
const batchRunId = toBatchRunId(batchId);
|
||||
if (!batchRunId) return null;
|
||||
|
||||
const batchRun = await store.findBatchTaskRunById(batchRunId);
|
||||
if (!batchRun) return null;
|
||||
|
||||
return (await userCanAccessEnvironment(prisma, batchRun.runtimeEnvironmentId, userId))
|
||||
? batchRun.id
|
||||
: null;
|
||||
}
|
||||
|
||||
function toBatchRunId(batchId: string): string | null {
|
||||
try {
|
||||
return BatchId.toId(batchId);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function userCanAccessEnvironment(
|
||||
prisma: PrismaClientOrTransaction,
|
||||
runtimeEnvironmentId: string,
|
||||
userId: string
|
||||
): Promise<boolean> {
|
||||
const environment = await prisma.runtimeEnvironment.findFirst({
|
||||
where: {
|
||||
id: runtimeEnvironmentId,
|
||||
organization: { members: { some: { userId } } },
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
return !!environment;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,231 @@
|
||||
import { BulkActionId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
type Prisma,
|
||||
BulkActionNotificationType,
|
||||
BulkActionStatus,
|
||||
BulkActionType,
|
||||
type PrismaClient,
|
||||
type TaskRunStatus,
|
||||
} from "@trigger.dev/database";
|
||||
import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
|
||||
import { prisma } from "~/db.server";
|
||||
import type { RunsRepository } from "~/services/runsRepository/runsRepository.server";
|
||||
import {
|
||||
countInProgressRunsForBillableEnvironment,
|
||||
countQueuedRunsForBillableEnvironment,
|
||||
createBillingLimitRunsRepository,
|
||||
getBillableEnvironmentsForBillingLimit,
|
||||
} from "./billingLimitQueuedRuns.server";
|
||||
import { BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS } from "./billingLimitConstants";
|
||||
|
||||
export const BILLING_LIMIT_RESOLVE_CANCEL_SOURCE = "billing_limit_resolve_new_only";
|
||||
export const BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE = "billing_limit_in_progress";
|
||||
|
||||
export class BillingLimitBulkCancelIncompleteError extends Error {
|
||||
constructor(readonly bulkActionId: string) {
|
||||
super(`Billing limit bulk cancel did not complete within time budget: ${bulkActionId}`);
|
||||
this.name = "BillingLimitBulkCancelIncompleteError";
|
||||
}
|
||||
}
|
||||
|
||||
type BulkCancelSource =
|
||||
| typeof BILLING_LIMIT_RESOLVE_CANCEL_SOURCE
|
||||
| typeof BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE;
|
||||
|
||||
export type BillingLimitBulkCancelDeps = {
|
||||
prismaClient?: PrismaClient;
|
||||
createRunsRepository?: (organizationId: string) => Promise<RunsRepository>;
|
||||
enqueueProcessBulkAction?: (bulkActionId: string) => Promise<unknown>;
|
||||
processBulkActionToCompletion?: (
|
||||
bulkActionId: string,
|
||||
options?: { deadline?: number }
|
||||
) => Promise<{ completed: boolean }>;
|
||||
};
|
||||
|
||||
function resolveBulkCancelDeps(deps?: BillingLimitBulkCancelDeps) {
|
||||
return {
|
||||
prismaClient: deps?.prismaClient ?? prisma,
|
||||
createRunsRepository: deps?.createRunsRepository ?? createBillingLimitRunsRepository,
|
||||
enqueueProcessBulkAction:
|
||||
deps?.enqueueProcessBulkAction ??
|
||||
(async (bulkActionId: string) => {
|
||||
// Imported dynamically for the same reason as BulkActionService below:
|
||||
// commonWorker.server transitively loads marqs -> the
|
||||
// TaskRunConcurrencyTracker singleton, which throws when REDIS_HOST/
|
||||
// REDIS_PORT are unset (e.g. the webapp unit-test CI job).
|
||||
const { commonWorker } = await import("~/v3/commonWorker.server");
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${bulkActionId}`,
|
||||
job: "processBulkAction",
|
||||
payload: { bulkActionId },
|
||||
});
|
||||
}),
|
||||
processBulkActionToCompletion:
|
||||
deps?.processBulkActionToCompletion ??
|
||||
(async (bulkActionId: string, options?: { deadline?: number }) => {
|
||||
// Imported dynamically so this module doesn't eagerly load BulkActionV2 ->
|
||||
// CancelTaskRunService -> marqs -> the TaskRunConcurrencyTracker singleton,
|
||||
// which throws when REDIS_HOST/REDIS_PORT are unset (e.g. the webapp
|
||||
// unit-test CI job).
|
||||
const { BulkActionService } = await import("~/v3/services/bulk/BulkActionV2.server");
|
||||
const service = new BulkActionService();
|
||||
return service.processToCompletion(bulkActionId, { deadline: options?.deadline });
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export class BillingLimitBulkCancelService {
|
||||
static async cancelQueuedRuns(
|
||||
organizationId: string,
|
||||
options?: {
|
||||
dedupeKey?: string;
|
||||
waitForCompletion?: boolean;
|
||||
bulkCancelDeadline?: number;
|
||||
},
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return this.cancelRunsForBillableEnvironments(
|
||||
organizationId,
|
||||
{
|
||||
source: BILLING_LIMIT_RESOLVE_CANCEL_SOURCE,
|
||||
statuses: [...QUEUED_STATUSES],
|
||||
name: "Billing limit resolve — cancel queued runs",
|
||||
countRuns: countQueuedRunsForBillableEnvironment,
|
||||
dedupeKey: options?.dedupeKey,
|
||||
waitForCompletion: options?.waitForCompletion,
|
||||
bulkCancelDeadline: options?.bulkCancelDeadline,
|
||||
},
|
||||
deps
|
||||
);
|
||||
}
|
||||
|
||||
static async cancelInProgressRuns(
|
||||
organizationId: string,
|
||||
options: { hitAt: string },
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return this.cancelRunsForBillableEnvironments(
|
||||
organizationId,
|
||||
{
|
||||
source: BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE,
|
||||
statuses: [...RUNNING_STATUSES],
|
||||
name: "Billing limit hit — cancel in-progress runs",
|
||||
countRuns: countInProgressRunsForBillableEnvironment,
|
||||
dedupeKey: options.hitAt,
|
||||
},
|
||||
deps
|
||||
);
|
||||
}
|
||||
|
||||
private static async cancelRunsForBillableEnvironments(
|
||||
organizationId: string,
|
||||
options: {
|
||||
source: BulkCancelSource;
|
||||
statuses: TaskRunStatus[];
|
||||
name: string;
|
||||
countRuns: typeof countQueuedRunsForBillableEnvironment;
|
||||
dedupeKey?: string;
|
||||
waitForCompletion?: boolean;
|
||||
bulkCancelDeadline?: number;
|
||||
},
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
const {
|
||||
prismaClient,
|
||||
createRunsRepository,
|
||||
enqueueProcessBulkAction,
|
||||
processBulkActionToCompletion,
|
||||
} = resolveBulkCancelDeps(deps);
|
||||
|
||||
const environments = await getBillableEnvironmentsForBillingLimit(organizationId, prismaClient);
|
||||
|
||||
if (environments.length === 0) {
|
||||
return { bulkActionIds: [] };
|
||||
}
|
||||
|
||||
const runsRepository = await createRunsRepository(organizationId);
|
||||
const bulkActionIds: string[] = [];
|
||||
const bulkActionInternalIds: string[] = [];
|
||||
|
||||
for (const environment of environments) {
|
||||
if (options.dedupeKey) {
|
||||
const existing = await prismaClient.bulkActionGroup.findFirst({
|
||||
where: {
|
||||
environmentId: environment.id,
|
||||
type: BulkActionType.CANCEL,
|
||||
dedupeKey: options.dedupeKey,
|
||||
status: { not: BulkActionStatus.ABORTED },
|
||||
},
|
||||
select: { id: true, friendlyId: true, status: true },
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
bulkActionIds.push(existing.friendlyId);
|
||||
|
||||
if (existing.status === BulkActionStatus.COMPLETED) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
bulkActionInternalIds.push(existing.id);
|
||||
} else {
|
||||
await enqueueProcessBulkAction(existing.id);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const count = await options.countRuns(runsRepository, organizationId, environment);
|
||||
|
||||
if (count === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { id, friendlyId } = BulkActionId.generate();
|
||||
|
||||
await prismaClient.bulkActionGroup.create({
|
||||
data: {
|
||||
id,
|
||||
friendlyId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
name: options.name,
|
||||
type: BulkActionType.CANCEL,
|
||||
dedupeKey: options.dedupeKey,
|
||||
params: {
|
||||
statuses: options.statuses,
|
||||
finalizeRun: true,
|
||||
source: options.source,
|
||||
...(options.dedupeKey ? { dedupeKey: options.dedupeKey } : {}),
|
||||
} as Prisma.InputJsonValue,
|
||||
queryName: "bulk_action_v1",
|
||||
totalCount: count,
|
||||
completionNotification: BulkActionNotificationType.NONE,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
bulkActionInternalIds.push(id);
|
||||
} else {
|
||||
await enqueueProcessBulkAction(id);
|
||||
}
|
||||
|
||||
bulkActionIds.push(friendlyId);
|
||||
}
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
const deadline =
|
||||
options.bulkCancelDeadline ?? Date.now() + BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS;
|
||||
|
||||
for (const bulkActionId of bulkActionInternalIds) {
|
||||
const result = await processBulkActionToCompletion(bulkActionId, { deadline });
|
||||
if (!result.completed) {
|
||||
throw new BillingLimitBulkCancelIncompleteError(bulkActionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { bulkActionIds };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server";
|
||||
|
||||
export async function runBillingLimitCancelInProgressRuns(
|
||||
organizationId: string,
|
||||
hitAt: string
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return BillingLimitBulkCancelService.cancelInProgressRuns(organizationId, { hitAt });
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { RuntimeEnvironmentType } from "@trigger.dev/database";
|
||||
|
||||
export const BILLABLE_ENVIRONMENT_TYPES = [
|
||||
"PRODUCTION",
|
||||
"STAGING",
|
||||
"PREVIEW",
|
||||
] as const satisfies RuntimeEnvironmentType[];
|
||||
|
||||
export type BillableEnvironmentType = (typeof BILLABLE_ENVIRONMENT_TYPES)[number];
|
||||
|
||||
export const BILLING_LIMIT_CONVERGE_BATCH_SIZE = 50;
|
||||
|
||||
/** Max concurrent per-org billing limit lookups during reconciliation. */
|
||||
export const BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY = 10;
|
||||
|
||||
/** Inline bulk-cancel budget for billing limit resolve (worker visibility is 10 min). */
|
||||
export const BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS = 8 * 60_000;
|
||||
|
||||
export type BillingLimitConvergeTargetState = "grace" | "rejected" | "ok";
|
||||
|
||||
export function isBillableEnvironmentType(type: RuntimeEnvironmentType): boolean {
|
||||
return (BILLABLE_ENVIRONMENT_TYPES as readonly RuntimeEnvironmentType[]).includes(type);
|
||||
}
|
||||
|
||||
export function buildBillingLimitResolveDedupeKey(
|
||||
organizationId: string,
|
||||
resolvedAt: string
|
||||
): string {
|
||||
return `billing-limit-resolve:${organizationId}:${resolvedAt}`;
|
||||
}
|
||||
|
||||
export function buildBillingLimitResolveJobId(organizationId: string, resolvedAt: string): string {
|
||||
return `billingLimit.resolve:${organizationId}:${resolvedAt}`;
|
||||
}
|
||||
|
||||
export function buildBillingLimitInProgressCancelJobId(
|
||||
organizationId: string,
|
||||
hitAt: string
|
||||
): string {
|
||||
return `billingLimit.cancelInProgress:${organizationId}:${hitAt}`;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import {
|
||||
EnvironmentPauseSource,
|
||||
type Organization,
|
||||
type PrismaClient,
|
||||
type Project,
|
||||
type RuntimeEnvironment,
|
||||
} from "@trigger.dev/database";
|
||||
import { prisma } from "~/db.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import {
|
||||
BILLABLE_ENVIRONMENT_TYPES,
|
||||
BILLING_LIMIT_CONVERGE_BATCH_SIZE,
|
||||
type BillingLimitConvergeTargetState,
|
||||
} from "./billingLimitConstants";
|
||||
|
||||
export type ConvergeOrgResult = {
|
||||
paused: number;
|
||||
unpaused: number;
|
||||
};
|
||||
|
||||
type EnvironmentWithRelations = RuntimeEnvironment & {
|
||||
organization: Organization;
|
||||
project: Project;
|
||||
};
|
||||
|
||||
type UpdateEnvConcurrency = (
|
||||
environment: EnvironmentWithRelations,
|
||||
maximumConcurrencyLimit?: number
|
||||
) => Promise<void>;
|
||||
|
||||
export async function convergeBillingLimitEnvironmentsForOrg(
|
||||
organizationId: string,
|
||||
targetState: BillingLimitConvergeTargetState,
|
||||
options?: {
|
||||
batchSize?: number;
|
||||
prismaClient?: PrismaClient;
|
||||
updateConcurrency?: UpdateEnvConcurrency;
|
||||
}
|
||||
): Promise<ConvergeOrgResult> {
|
||||
const db = options?.prismaClient ?? prisma;
|
||||
const batchSize = options?.batchSize ?? BILLING_LIMIT_CONVERGE_BATCH_SIZE;
|
||||
// Imported dynamically so this module (reachable from upsertBranch.server.ts at
|
||||
// module load) doesn't eagerly load runQueue.server -> marqs -> triggerTaskV1 ->
|
||||
// the autoIncrementCounter singleton, which throws when REDIS_HOST/REDIS_PORT are
|
||||
// unset (e.g. the webapp unit-test CI job).
|
||||
const updateConcurrency =
|
||||
options?.updateConcurrency ??
|
||||
(async (environment, maximumConcurrencyLimit) => {
|
||||
const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server");
|
||||
return updateEnvConcurrencyLimits(environment, maximumConcurrencyLimit);
|
||||
});
|
||||
|
||||
if (targetState === "ok") {
|
||||
return unpauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency);
|
||||
}
|
||||
|
||||
return pauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency);
|
||||
}
|
||||
|
||||
async function pauseBillingLimitEnvironments(
|
||||
organizationId: string,
|
||||
db: PrismaClient,
|
||||
batchSize: number,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
): Promise<ConvergeOrgResult> {
|
||||
let paused = 0;
|
||||
let cursor: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const environments = await db.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
type: { in: [...BILLABLE_ENVIRONMENT_TYPES] },
|
||||
paused: false,
|
||||
},
|
||||
take: batchSize,
|
||||
...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
|
||||
orderBy: { id: "asc" },
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (environments.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const environment of environments) {
|
||||
await pauseEnvironmentForBillingLimit(environment, db, updateConcurrency);
|
||||
paused++;
|
||||
}
|
||||
|
||||
cursor = environments[environments.length - 1]?.id;
|
||||
if (environments.length < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Billing limit converge paused environments", {
|
||||
organizationId,
|
||||
paused,
|
||||
});
|
||||
|
||||
return { paused, unpaused: 0 };
|
||||
}
|
||||
|
||||
async function unpauseBillingLimitEnvironments(
|
||||
organizationId: string,
|
||||
db: PrismaClient,
|
||||
batchSize: number,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
): Promise<ConvergeOrgResult> {
|
||||
let unpaused = 0;
|
||||
let cursor: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const environments = await db.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
take: batchSize,
|
||||
...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
|
||||
orderBy: { id: "asc" },
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (environments.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const environment of environments) {
|
||||
await resumeEnvironmentFromBillingLimit(environment, db, updateConcurrency);
|
||||
unpaused++;
|
||||
}
|
||||
|
||||
cursor = environments[environments.length - 1]?.id;
|
||||
if (environments.length < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Billing limit converge unpaused environments", {
|
||||
organizationId,
|
||||
unpaused,
|
||||
});
|
||||
|
||||
return { paused: 0, unpaused };
|
||||
}
|
||||
|
||||
async function pauseEnvironmentForBillingLimit(
|
||||
environment: EnvironmentWithRelations,
|
||||
db: PrismaClient,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
) {
|
||||
const updated = await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await updateConcurrency(updated, 0);
|
||||
} catch (error) {
|
||||
await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: { paused: false, pauseSource: null },
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
// The env's paused state changed (or was rolled back); drop any cached copy either way.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeEnvironmentFromBillingLimit(
|
||||
environment: EnvironmentWithRelations,
|
||||
db: PrismaClient,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
) {
|
||||
const updated = await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: false,
|
||||
pauseSource: null,
|
||||
},
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await updateConcurrency(updated);
|
||||
} catch (error) {
|
||||
await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
// The env's paused state changed (or was rolled back); drop any cached copy either way.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import { z } from "zod";
|
||||
import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server";
|
||||
import { runBillingLimitReconcileTick } from "./runBillingLimitReconcileTick.server";
|
||||
import { seedBillingLimitReconcileQueue } from "./billingLimitReconcileQueue.server";
|
||||
|
||||
const ConvergePayloadSchema = z.object({
|
||||
organizationId: z.string(),
|
||||
targetState: z.enum(["grace", "rejected", "ok"]),
|
||||
});
|
||||
|
||||
export class BillingLimitConvergeEnvironmentsService {
|
||||
static async seedReconcileQueue(organizationId: string) {
|
||||
await seedBillingLimitReconcileQueue(organizationId);
|
||||
}
|
||||
|
||||
static async runConverge(payload: z.infer<typeof ConvergePayloadSchema>) {
|
||||
const parsed = ConvergePayloadSchema.parse(payload);
|
||||
return convergeBillingLimitEnvironmentsForOrg(parsed.organizationId, parsed.targetState);
|
||||
}
|
||||
|
||||
static async runReconcileTick() {
|
||||
await runBillingLimitReconcileTick();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { bustBillingLimitCaches } from "~/services/platform.v3.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server";
|
||||
import { buildBillingLimitResolveDedupeKey } from "./billingLimitConstants";
|
||||
import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export async function convergeBillingLimitResolve(
|
||||
pending: PendingBillingLimitResolve
|
||||
): Promise<void> {
|
||||
const { organizationId, resumeMode, resolvedAt } = pending;
|
||||
|
||||
bustBillingLimitCaches(organizationId);
|
||||
|
||||
if (resumeMode === "new_only") {
|
||||
await BillingLimitBulkCancelService.cancelQueuedRuns(organizationId, {
|
||||
dedupeKey: buildBillingLimitResolveDedupeKey(organizationId, resolvedAt),
|
||||
waitForCompletion: true,
|
||||
});
|
||||
}
|
||||
|
||||
await convergeBillingLimitEnvironmentsForOrg(organizationId, "ok");
|
||||
|
||||
logger.info("Converged billing limit resolve", {
|
||||
organizationId,
|
||||
resumeMode,
|
||||
resolvedAt,
|
||||
});
|
||||
}
|
||||
|
||||
export { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server";
|
||||
@@ -0,0 +1,26 @@
|
||||
export type BillingLimitHitPayload = {
|
||||
organizationId: string;
|
||||
hitAt: string;
|
||||
cancelInProgressRuns: boolean;
|
||||
};
|
||||
|
||||
export type BillingLimitHitDeps = {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
seedReconcileQueue: (organizationId: string) => Promise<void>;
|
||||
enqueueConverge: (organizationId: string, targetState: "grace") => Promise<unknown>;
|
||||
enqueueCancelInProgressRuns: (organizationId: string, hitAt: string) => Promise<unknown>;
|
||||
};
|
||||
|
||||
/** Process billing limit grace hit from the billing platform webhook. */
|
||||
export async function processBillingLimitHit(
|
||||
payload: BillingLimitHitPayload,
|
||||
deps: BillingLimitHitDeps
|
||||
): Promise<void> {
|
||||
deps.bustCaches(payload.organizationId);
|
||||
await deps.seedReconcileQueue(payload.organizationId);
|
||||
await deps.enqueueConverge(payload.organizationId, "grace");
|
||||
|
||||
if (payload.cancelInProgressRuns) {
|
||||
await deps.enqueueCancelInProgressRuns(payload.organizationId, payload.hitAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type PendingBillingLimitResolve = {
|
||||
organizationId: string;
|
||||
resumeMode: "queue" | "new_only";
|
||||
resolvedAt: string;
|
||||
};
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { classifyPendingBillingLimitResolveConvergeFailure } from "./billingLimitPendingResolveFailure.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type RunPendingBillingLimitResolveDeps = {
|
||||
converge?: (pending: PendingBillingLimitResolve) => Promise<void>;
|
||||
complete?: (organizationId: string) => Promise<{ completed: boolean } | undefined>;
|
||||
};
|
||||
|
||||
export async function runPendingBillingLimitResolves(
|
||||
pendingResolves: PendingBillingLimitResolve[],
|
||||
deps: RunPendingBillingLimitResolveDeps = {}
|
||||
): Promise<Set<string>> {
|
||||
const converge =
|
||||
deps.converge ??
|
||||
(await import("./billingLimitConvergeResolve.server")).convergeBillingLimitResolve;
|
||||
const complete =
|
||||
deps.complete ?? (await import("~/services/platform.v3.server")).completeBillingLimitResolve;
|
||||
|
||||
const stillPendingOrgIds = new Set<string>();
|
||||
|
||||
for (const pending of pendingResolves) {
|
||||
try {
|
||||
await converge(pending);
|
||||
} catch (error) {
|
||||
logger.error("Failed to converge pending billing limit resolve", {
|
||||
failureClass: classifyPendingBillingLimitResolveConvergeFailure(pending.resumeMode),
|
||||
error,
|
||||
organizationId: pending.organizationId,
|
||||
resumeMode: pending.resumeMode,
|
||||
resolvedAt: pending.resolvedAt,
|
||||
});
|
||||
stillPendingOrgIds.add(pending.organizationId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const completion = await complete(pending.organizationId);
|
||||
if (!completion || completion.completed !== true) {
|
||||
throw new Error("Billing platform client unavailable");
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Failed to ack pending billing limit resolve", {
|
||||
failureClass: "ack-only",
|
||||
error,
|
||||
organizationId: pending.organizationId,
|
||||
resumeMode: pending.resumeMode,
|
||||
resolvedAt: pending.resolvedAt,
|
||||
});
|
||||
stillPendingOrgIds.add(pending.organizationId);
|
||||
}
|
||||
}
|
||||
|
||||
return stillPendingOrgIds;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
export type PendingBillingLimitResolveFailureClass =
|
||||
| "cancel-failing"
|
||||
| "converge-failing"
|
||||
| "ack-only";
|
||||
|
||||
/** Used in converge logs to classify stuck pending resolves. */
|
||||
export function classifyPendingBillingLimitResolveConvergeFailure(
|
||||
resumeMode: "queue" | "new_only"
|
||||
): Exclude<PendingBillingLimitResolveFailureClass, "ack-only"> {
|
||||
return resumeMode === "new_only" ? "cancel-failing" : "converge-failing";
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database";
|
||||
import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
|
||||
import { prisma } from "~/db.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
|
||||
import { BILLABLE_ENVIRONMENT_TYPES } from "./billingLimitConstants";
|
||||
|
||||
export type BillableEnvironmentRef = {
|
||||
id: string;
|
||||
projectId: string;
|
||||
};
|
||||
|
||||
export async function getBillableEnvironmentsForBillingLimit(
|
||||
organizationId: string,
|
||||
prismaClient: PrismaClient = prisma
|
||||
): Promise<BillableEnvironmentRef[]> {
|
||||
return prismaClient.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
type: { in: [...BILLABLE_ENVIRONMENT_TYPES] },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
projectId: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function createBillingLimitRunsRepository(organizationId: string) {
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"standard"
|
||||
);
|
||||
|
||||
return new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: prisma as PrismaClient,
|
||||
});
|
||||
}
|
||||
|
||||
export async function countQueuedRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef
|
||||
): Promise<number> {
|
||||
return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [
|
||||
...QUEUED_STATUSES,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function countInProgressRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef
|
||||
): Promise<number> {
|
||||
return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [
|
||||
...RUNNING_STATUSES,
|
||||
]);
|
||||
}
|
||||
|
||||
async function countRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef,
|
||||
statuses: TaskRunStatus[]
|
||||
): Promise<number> {
|
||||
return runsRepository.countRuns({
|
||||
organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
statuses,
|
||||
});
|
||||
}
|
||||
|
||||
/** Same source as BillingLimitBulkCancelService — ClickHouse countRuns(QUEUED_STATUSES). */
|
||||
export async function countBillableQueuedRunsForOrganization(
|
||||
organizationId: string
|
||||
): Promise<number> {
|
||||
const environments = await getBillableEnvironmentsForBillingLimit(organizationId);
|
||||
|
||||
if (environments.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const runsRepository = await createBillingLimitRunsRepository(organizationId);
|
||||
|
||||
let total = 0;
|
||||
|
||||
for (const environment of environments) {
|
||||
total += await countQueuedRunsForBillableEnvironment(
|
||||
runsRepository,
|
||||
organizationId,
|
||||
environment
|
||||
);
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { env } from "~/env.server";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
const RECONCILE_QUEUE_KEY = "billing-limit:reconcile-queue";
|
||||
|
||||
function createQueueRedis() {
|
||||
return createRedisClient("billing-limit:reconcile", {
|
||||
keyPrefix: "",
|
||||
host: env.BILLING_LIMIT_WORKER_REDIS_HOST,
|
||||
port: env.BILLING_LIMIT_WORKER_REDIS_PORT,
|
||||
username: env.BILLING_LIMIT_WORKER_REDIS_USERNAME,
|
||||
password: env.BILLING_LIMIT_WORKER_REDIS_PASSWORD,
|
||||
tlsDisabled: env.BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED === "true",
|
||||
});
|
||||
}
|
||||
|
||||
const queueRedis = singleton("billingLimitReconcileQueueRedis", createQueueRedis);
|
||||
|
||||
export async function seedBillingLimitReconcileQueue(organizationId: string): Promise<void> {
|
||||
await queueRedis.sadd(RECONCILE_QUEUE_KEY, organizationId);
|
||||
}
|
||||
|
||||
export async function readBillingLimitReconcileQueue(): Promise<string[]> {
|
||||
return queueRedis.smembers(RECONCILE_QUEUE_KEY);
|
||||
}
|
||||
|
||||
export async function removeFromBillingLimitReconcileQueue(
|
||||
organizationIds: string[]
|
||||
): Promise<void> {
|
||||
if (organizationIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
await queueRedis.srem(RECONCILE_QUEUE_KEY, ...organizationIds);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { BillingLimitConvergeTargetState } from "./billingLimitConstants";
|
||||
import type { OrgReconcileTarget } from "./billingLimitReconciliation.server";
|
||||
|
||||
export async function reconcileBillingLimitTarget(
|
||||
target: OrgReconcileTarget,
|
||||
deps: {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
enqueueConverge: (
|
||||
organizationId: string,
|
||||
targetState: BillingLimitConvergeTargetState
|
||||
) => Promise<unknown>;
|
||||
}
|
||||
) {
|
||||
// Safety net when webhooks are lost: bust stale entitlement after reject or resolve.
|
||||
if (target.targetState === "rejected" || target.targetState === "ok") {
|
||||
deps.bustCaches(target.organizationId);
|
||||
}
|
||||
|
||||
await deps.enqueueConverge(target.organizationId, target.targetState);
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import pMap from "p-map";
|
||||
import { prisma } from "~/db.server";
|
||||
import type { BillingLimitResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getActiveBillingLimits, getBillingLimit } from "~/services/platform.v3.server";
|
||||
import {
|
||||
BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY,
|
||||
type BillingLimitConvergeTargetState,
|
||||
} from "./billingLimitConstants";
|
||||
import {
|
||||
readBillingLimitReconcileQueue,
|
||||
removeFromBillingLimitReconcileQueue,
|
||||
} from "./billingLimitReconcileQueue.server";
|
||||
|
||||
export type OrgReconcileTarget = {
|
||||
organizationId: string;
|
||||
targetState: BillingLimitConvergeTargetState;
|
||||
};
|
||||
|
||||
export function resolveConvergeTargetFromBillingLimit(
|
||||
billingLimit: BillingLimitResult | undefined
|
||||
): BillingLimitConvergeTargetState {
|
||||
if (!billingLimit?.isConfigured) {
|
||||
return "ok";
|
||||
}
|
||||
|
||||
if (billingLimit.limitState.status === "grace") {
|
||||
return "grace";
|
||||
}
|
||||
|
||||
if (billingLimit.limitState.status === "rejected") {
|
||||
return "rejected";
|
||||
}
|
||||
|
||||
return "ok";
|
||||
}
|
||||
|
||||
/** Reconcile path only — skip org when the platform lookup failed (undefined ≠ unconfigured). */
|
||||
export function resolveReconcileTargetFromBillingLimit(
|
||||
billingLimit: BillingLimitResult | undefined
|
||||
): BillingLimitConvergeTargetState | undefined {
|
||||
if (billingLimit === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return resolveConvergeTargetFromBillingLimit(billingLimit);
|
||||
}
|
||||
|
||||
export async function getOrgIdsWithBillingPauseSource(): Promise<string[]> {
|
||||
const rows = await prisma.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
select: {
|
||||
organizationId: true,
|
||||
},
|
||||
distinct: ["organizationId"],
|
||||
});
|
||||
|
||||
return rows.map((row) => row.organizationId);
|
||||
}
|
||||
|
||||
export function collectOrgIdsNeedingBillingLimitLookup(options: {
|
||||
staleOrgIds: string[];
|
||||
queuedOrgIds: string[];
|
||||
excludeOrgIds: Set<string>;
|
||||
coveredOrgIds: Set<string>;
|
||||
}): string[] {
|
||||
const orgIds = new Set<string>();
|
||||
|
||||
for (const organizationId of [...options.staleOrgIds, ...options.queuedOrgIds]) {
|
||||
if (options.excludeOrgIds.has(organizationId) || options.coveredOrgIds.has(organizationId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
orgIds.add(organizationId);
|
||||
}
|
||||
|
||||
return [...orgIds];
|
||||
}
|
||||
|
||||
export async function resolveReconcileTargetsForOrgLookups(
|
||||
organizationIds: string[],
|
||||
options?: {
|
||||
getBillingLimit?: (organizationId: string) => Promise<BillingLimitResult | undefined>;
|
||||
concurrency?: number;
|
||||
}
|
||||
): Promise<Map<string, BillingLimitConvergeTargetState>> {
|
||||
const lookupBillingLimit = options?.getBillingLimit ?? getBillingLimit;
|
||||
const concurrency = options?.concurrency ?? BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY;
|
||||
const targets = new Map<string, BillingLimitConvergeTargetState>();
|
||||
|
||||
await pMap(
|
||||
organizationIds,
|
||||
async (organizationId) => {
|
||||
try {
|
||||
const billingLimit = await lookupBillingLimit(organizationId);
|
||||
const targetState = resolveReconcileTargetFromBillingLimit(billingLimit);
|
||||
if (targetState === undefined) {
|
||||
logger.warn("Skipping billing limit reconcile — platform lookup unavailable", {
|
||||
organizationId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
targets.set(organizationId, targetState);
|
||||
} catch (error) {
|
||||
logger.error("Failed billing limit lookup for reconcile", {
|
||||
organizationId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
},
|
||||
{ concurrency }
|
||||
);
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
export async function collectOrgsToReconcile(options?: { excludeOrgIds?: Set<string> }): Promise<{
|
||||
targets: OrgReconcileTarget[];
|
||||
queuedOrgIds: string[];
|
||||
}> {
|
||||
const excludeOrgIds = options?.excludeOrgIds ?? new Set<string>();
|
||||
const targetByOrgId = new Map<string, BillingLimitConvergeTargetState>();
|
||||
|
||||
const activeLimits = await getActiveBillingLimits();
|
||||
if (activeLimits) {
|
||||
for (const org of activeLimits.orgs) {
|
||||
if (excludeOrgIds.has(org.orgId)) {
|
||||
continue;
|
||||
}
|
||||
targetByOrgId.set(org.orgId, org.limitState);
|
||||
}
|
||||
}
|
||||
|
||||
const [staleOrgIds, queuedOrgIds] = await Promise.all([
|
||||
getOrgIdsWithBillingPauseSource(),
|
||||
readBillingLimitReconcileQueue(),
|
||||
]);
|
||||
|
||||
const orgIdsNeedingLookup = collectOrgIdsNeedingBillingLimitLookup({
|
||||
staleOrgIds,
|
||||
queuedOrgIds,
|
||||
excludeOrgIds,
|
||||
coveredOrgIds: new Set(targetByOrgId.keys()),
|
||||
});
|
||||
|
||||
const lookedUpTargets = await resolveReconcileTargetsForOrgLookups(orgIdsNeedingLookup);
|
||||
for (const [organizationId, targetState] of lookedUpTargets) {
|
||||
targetByOrgId.set(organizationId, targetState);
|
||||
}
|
||||
|
||||
return {
|
||||
targets: Array.from(targetByOrgId.entries()).map(([organizationId, targetState]) => ({
|
||||
organizationId,
|
||||
targetState,
|
||||
})),
|
||||
queuedOrgIds,
|
||||
};
|
||||
}
|
||||
|
||||
export async function clearProcessedReconcileQueueEntries(
|
||||
queuedOrgIds: string[],
|
||||
processedOrgIds: string[]
|
||||
): Promise<void> {
|
||||
const processed = new Set(processedOrgIds);
|
||||
const toRemove = queuedOrgIds.filter((orgId) => processed.has(orgId));
|
||||
await removeFromBillingLimitReconcileQueue(toRemove);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type BillingLimitResolveDeps = {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
enqueueResolve: (pending: PendingBillingLimitResolve) => Promise<unknown>;
|
||||
};
|
||||
|
||||
/** Process billing limit resolve from the billing platform webhook. */
|
||||
export async function processBillingLimitResolve(
|
||||
pending: PendingBillingLimitResolve,
|
||||
deps: BillingLimitResolveDeps
|
||||
): Promise<void> {
|
||||
deps.bustCaches(pending.organizationId);
|
||||
await deps.enqueueResolve(pending);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import { prisma } from "~/db.server";
|
||||
import { countBillableQueuedRunsForOrganization } from "./billingLimitQueuedRuns.server";
|
||||
|
||||
export async function getBillingLimitQueuedRunCount(organizationId: string): Promise<number> {
|
||||
return countBillableQueuedRunsForOrganization(organizationId);
|
||||
}
|
||||
|
||||
export async function countBillingLimitPausedEnvironments(organizationId: string): Promise<number> {
|
||||
return prisma.runtimeEnvironment.count({
|
||||
where: {
|
||||
organizationId,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
});
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import {
|
||||
EnvironmentPauseSource,
|
||||
type Organization,
|
||||
type Project,
|
||||
type RuntimeEnvironment,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import type { BillingLimitResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { isBillableEnvironmentType } from "./billingLimitConstants";
|
||||
import { resolveConvergeTargetFromBillingLimit } from "./billingLimitReconciliation.server";
|
||||
|
||||
export type InitialEnvPauseState = {
|
||||
paused: boolean;
|
||||
pauseSource: typeof EnvironmentPauseSource.BILLING_LIMIT | null;
|
||||
};
|
||||
|
||||
export type GetInitialEnvPauseStateDeps = {
|
||||
getBillingLimit?: (organizationId: string) => Promise<BillingLimitResult | undefined>;
|
||||
};
|
||||
|
||||
export async function getInitialEnvPauseStateForBillingLimit(
|
||||
organizationId: string,
|
||||
type: RuntimeEnvironmentType,
|
||||
deps: GetInitialEnvPauseStateDeps = {}
|
||||
): Promise<InitialEnvPauseState> {
|
||||
if (!isBillableEnvironmentType(type)) {
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
let billingLimit: BillingLimitResult | undefined;
|
||||
try {
|
||||
billingLimit = deps.getBillingLimit
|
||||
? await deps.getBillingLimit(organizationId)
|
||||
: await (await import("~/services/platform.v3.server")).getBillingLimit(organizationId);
|
||||
} catch (error) {
|
||||
logger.error("Failed to fetch billing limit for initial env pause state", {
|
||||
organizationId,
|
||||
error,
|
||||
});
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
const targetState = resolveConvergeTargetFromBillingLimit(billingLimit);
|
||||
|
||||
if (targetState === "grace" || targetState === "rejected") {
|
||||
return {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
};
|
||||
}
|
||||
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
export async function applyBillingLimitPauseAfterEnvCreate(
|
||||
environment: RuntimeEnvironment & { organization: Organization; project: Project }
|
||||
): Promise<void> {
|
||||
if (!environment.paused || environment.pauseSource !== EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Imported dynamically so this module (pulled in at module load by
|
||||
// upsertBranch.server.ts) doesn't eagerly load runQueue.server -> marqs ->
|
||||
// triggerTaskV1 -> the autoIncrementCounter singleton, which throws when
|
||||
// REDIS_HOST/REDIS_PORT are unset (e.g. the webapp unit-test CI job).
|
||||
const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server");
|
||||
await updateEnvConcurrencyLimits(environment, 0);
|
||||
} catch (error) {
|
||||
logger.error("Failed to apply billing-limit pause after env create", {
|
||||
environmentId: environment.id,
|
||||
organizationId: environment.organizationId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import type { PauseStatus } from "~/v3/services/pauseEnvironment.server";
|
||||
|
||||
/**
|
||||
* Guards manual pause/resume API calls while an environment is billing-paused.
|
||||
*
|
||||
* Design trade-off: billing-limit converge unpauses every environment with
|
||||
* `pauseSource=BILLING_LIMIT` on resolve. We therefore do not record a
|
||||
* separate manual pause on top of billing enforcement — a manual pause attempt
|
||||
* while already billing-paused is a silent no-op (`success: true`, still
|
||||
* paused). If the limit is later resolved, that environment is unpaused with
|
||||
* the rest, even if the caller intended to keep it paused.
|
||||
*
|
||||
* The queues UI hides pause/resume while `pauseSource=BILLING_LIMIT`; API
|
||||
* callers can still hit this path and should treat the no-op as idempotent.
|
||||
*/
|
||||
export function getManualPauseEnvironmentResult(
|
||||
action: PauseStatus,
|
||||
pauseSource: EnvironmentPauseSource | null | undefined
|
||||
):
|
||||
| { proceed: true }
|
||||
| { proceed: false; success: true; state: PauseStatus }
|
||||
| { proceed: false; success: false; error: string } {
|
||||
if (action === "resumed" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
return {
|
||||
proceed: false,
|
||||
success: false,
|
||||
error:
|
||||
"This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.",
|
||||
};
|
||||
}
|
||||
|
||||
if (action === "paused" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
// Already billing-paused; do not overwrite pauseSource so resolve converge
|
||||
// can still find and unpause this environment.
|
||||
return {
|
||||
proceed: false,
|
||||
success: true,
|
||||
state: "paused",
|
||||
};
|
||||
}
|
||||
|
||||
return { proceed: true };
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { BillingLimitsPendingResolvesResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
import type { OrgReconcileTarget } from "./billingLimitReconciliation.server";
|
||||
import type { reconcileBillingLimitTarget } from "./billingLimitReconcileTarget.server";
|
||||
|
||||
export type RunBillingLimitReconcileTickDeps = {
|
||||
getPendingResolves?: () => Promise<BillingLimitsPendingResolvesResult | undefined>;
|
||||
runPendingResolves?: (pendingResolves: PendingBillingLimitResolve[]) => Promise<Set<string>>;
|
||||
collectOrgs?: (options?: { excludeOrgIds?: Set<string> }) => Promise<{
|
||||
targets: OrgReconcileTarget[];
|
||||
queuedOrgIds: string[];
|
||||
}>;
|
||||
reconcileTarget?: typeof reconcileBillingLimitTarget;
|
||||
clearProcessedQueue?: (queuedOrgIds: string[], processedOrgIds: string[]) => Promise<void>;
|
||||
bustCaches?: (organizationId: string) => void;
|
||||
enqueueConverge?: (
|
||||
organizationId: string,
|
||||
targetState: OrgReconcileTarget["targetState"]
|
||||
) => Promise<void>;
|
||||
};
|
||||
|
||||
export async function runBillingLimitReconcileTick(
|
||||
deps: RunBillingLimitReconcileTickDeps = {}
|
||||
): Promise<void> {
|
||||
const getPendingResolves =
|
||||
deps.getPendingResolves ??
|
||||
(await import("~/services/platform.v3.server")).getPendingBillingLimitResolves;
|
||||
const runPendingResolves = deps.runPendingResolves ?? runPendingBillingLimitResolves;
|
||||
const collectOrgs =
|
||||
deps.collectOrgs ??
|
||||
(await import("./billingLimitReconciliation.server")).collectOrgsToReconcile;
|
||||
const reconcileTarget =
|
||||
deps.reconcileTarget ??
|
||||
(await import("./billingLimitReconcileTarget.server")).reconcileBillingLimitTarget;
|
||||
const clearProcessedQueue =
|
||||
deps.clearProcessedQueue ??
|
||||
(await import("./billingLimitReconciliation.server")).clearProcessedReconcileQueueEntries;
|
||||
const bustCaches =
|
||||
deps.bustCaches ?? (await import("~/services/platform.v3.server")).bustBillingLimitCaches;
|
||||
|
||||
const pendingResolves = (await getPendingResolves())?.orgs ?? [];
|
||||
const stillPendingOrgIds = await runPendingResolves(pendingResolves);
|
||||
|
||||
const { targets, queuedOrgIds } = await collectOrgs({
|
||||
excludeOrgIds: stillPendingOrgIds,
|
||||
});
|
||||
|
||||
const enqueueConverge =
|
||||
deps.enqueueConverge ??
|
||||
(async (organizationId, targetState) => {
|
||||
const { enqueueBillingLimitConverge } = await import("~/v3/billingLimitWorker.server");
|
||||
await enqueueBillingLimitConverge(organizationId, targetState);
|
||||
});
|
||||
|
||||
const processedOrgIds: string[] = [];
|
||||
for (const target of targets) {
|
||||
try {
|
||||
await reconcileTarget(target, {
|
||||
bustCaches,
|
||||
enqueueConverge,
|
||||
});
|
||||
processedOrgIds.push(target.organizationId);
|
||||
} catch (error) {
|
||||
logger.error("Failed to reconcile billing limit target", {
|
||||
organizationId: target.organizationId,
|
||||
targetState: target.targetState,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await clearProcessedQueue(queuedOrgIds, processedOrgIds);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
// Real PG14 (legacy replica) + PG17 (new) proof for the bulk batch read-through adapter.
|
||||
// We NEVER mock the DB: each closure runs a real `$queryRaw` against the passed container
|
||||
// (crossing the actual PG14↔PG17 boundary) then filters an in-memory seeded set by id —
|
||||
// mirroring readThrough.server.test.ts's `realRead`. The only injected fakes are throwing
|
||||
// spies asserting a store was NEVER touched.
|
||||
import { heteroPostgresTest } from "@internal/testcontainers";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import { hydrateRunsAcrossSeam } from "./BulkActionV2.batchReadThrough.server";
|
||||
|
||||
vi.setConfig({ testTimeout: 60_000 });
|
||||
|
||||
// 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency.
|
||||
const LEGACY_RUN_ID = "run_" + "a".repeat(25);
|
||||
const NEW_RUN_ID = "run_" + "b".repeat(24) + "01";
|
||||
|
||||
type Row = { id: string };
|
||||
|
||||
// Real read against the given container, then return rows for the ids present in `present`.
|
||||
async function realReadFiltered(
|
||||
client: PrismaReplicaClient,
|
||||
ids: string[],
|
||||
present: Set<string>
|
||||
): Promise<Row[]> {
|
||||
await client.$queryRaw<{ marker: number }[]>`SELECT 1 AS marker`;
|
||||
return ids.filter((id) => present.has(id)).map((id) => ({ id }));
|
||||
}
|
||||
|
||||
describe("hydrateRunsAcrossSeam (PG14 legacy replica + PG17 new)", () => {
|
||||
heteroPostgresTest(
|
||||
"(a) mixed page: NEW id from new, LEGACY id from legacy replica; new id never hits legacy",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const onNew = new Set([NEW_RUN_ID]);
|
||||
const onLegacy = new Set([LEGACY_RUN_ID]);
|
||||
|
||||
const readLegacyReplica = vi.fn(
|
||||
async (replica: PrismaReplicaClient, ids: string[]): Promise<Row[]> => {
|
||||
if (ids.includes(NEW_RUN_ID)) {
|
||||
throw new Error("legacy replica must never be probed for a NEW-residency id");
|
||||
}
|
||||
return realReadFiltered(replica, ids, onLegacy);
|
||||
}
|
||||
);
|
||||
|
||||
const rows = await hydrateRunsAcrossSeam<Row>({
|
||||
runIds: [NEW_RUN_ID, LEGACY_RUN_ID],
|
||||
readNew: (client, ids) => realReadFiltered(client, ids, onNew),
|
||||
readLegacyReplica,
|
||||
deps: {
|
||||
splitEnabled: true,
|
||||
newClient: prisma17 as unknown as PrismaReplicaClient,
|
||||
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
|
||||
},
|
||||
});
|
||||
|
||||
const ids = rows.map((r) => r.id).sort();
|
||||
expect(ids).toEqual([LEGACY_RUN_ID, NEW_RUN_ID].sort());
|
||||
expect(readLegacyReplica).toHaveBeenCalledTimes(1);
|
||||
// legacy was only probed for the legacy id
|
||||
expect(readLegacyReplica.mock.calls[0][1]).toEqual([LEGACY_RUN_ID]);
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"(c) passthrough: splitEnabled false reads only the single client; legacy never touched",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const onNew = new Set([NEW_RUN_ID, LEGACY_RUN_ID]);
|
||||
const throwingLegacy = vi.fn(async (): Promise<Row[]> => {
|
||||
throw new Error("readLegacyReplica must never run in single-DB mode");
|
||||
});
|
||||
const readNew = vi.fn((client: PrismaReplicaClient, ids: string[]) =>
|
||||
realReadFiltered(client, ids, onNew)
|
||||
);
|
||||
|
||||
const rows = await hydrateRunsAcrossSeam<Row>({
|
||||
runIds: [NEW_RUN_ID, LEGACY_RUN_ID],
|
||||
readNew,
|
||||
readLegacyReplica: throwingLegacy,
|
||||
deps: {
|
||||
splitEnabled: false,
|
||||
// single collapsed store (use prisma17 here as the "new"/primary analog)
|
||||
newClient: prisma17 as unknown as PrismaReplicaClient,
|
||||
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
|
||||
},
|
||||
});
|
||||
|
||||
const ids = rows.map((r) => r.id).sort();
|
||||
expect(ids).toEqual([LEGACY_RUN_ID, NEW_RUN_ID].sort());
|
||||
expect(readNew).toHaveBeenCalledTimes(1);
|
||||
expect(throwingLegacy).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Batch adapter over the per-id `readThroughRun` (see
|
||||
* `~/v3/runOpsMigration/readThrough.server.ts`). A bulk action processes a PAGE of
|
||||
* member run ids at once, so instead of N per-id round trips this reproduces the
|
||||
* per-id read-through ordering as SET reads:
|
||||
*
|
||||
* 1. single-DB passthrough (splitEnabled === false): ONE read against the collapsed
|
||||
* store, no residency classification, no legacy probe.
|
||||
* 2. split on: classify each id's residency via `ownerEngine`, read NEW for every id
|
||||
* that could be on new (residency NEW *and* legacy-candidates — read-through is
|
||||
* new-FIRST for legacy too), then probe the LEGACY READ REPLICA ONLY for the
|
||||
* legacy-candidates the new read missed.
|
||||
*
|
||||
* Like the per-id layer this NEVER touches a legacy primary/writer — there is no such
|
||||
* handle. An id is read from new OR legacy, never both: legacy is only probed for ids
|
||||
* new missed, so the returned set needs no dedupe.
|
||||
*/
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import {
|
||||
runOpsLegacyReplica as defaultLegacyReplica,
|
||||
runOpsNewReplica as defaultNewClient,
|
||||
} from "~/db.server";
|
||||
import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
|
||||
export type SeamReadDeps = {
|
||||
/**
|
||||
* Resolved boot constant. REQUIRED here — the caller resolves it once per
|
||||
* request via `isSplitEnabled()`; this adapter never awaits it itself.
|
||||
*/
|
||||
splitEnabled: boolean;
|
||||
newClient?: PrismaReplicaClient;
|
||||
legacyReplica?: PrismaReplicaClient;
|
||||
logger?: { warn: (m: string, meta?: unknown) => void };
|
||||
};
|
||||
|
||||
type HydrateRunsAcrossSeamInput<T> = {
|
||||
runIds: string[];
|
||||
readNew: (client: PrismaReplicaClient, ids: string[]) => Promise<T[]>;
|
||||
readLegacyReplica: (replica: PrismaReplicaClient, ids: string[]) => Promise<T[]>;
|
||||
deps: SeamReadDeps;
|
||||
};
|
||||
|
||||
/** Every row shape we hydrate carries an `id` (CANCEL select includes it; REPLAY is a full row). */
|
||||
function getId(row: unknown): string {
|
||||
return (row as { id: string }).id;
|
||||
}
|
||||
|
||||
export async function hydrateRunsAcrossSeam<T>(input: HydrateRunsAcrossSeamInput<T>): Promise<T[]> {
|
||||
const { runIds, deps } = input;
|
||||
|
||||
if (runIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const newClient = deps.newClient ?? defaultNewClient;
|
||||
|
||||
// Passthrough: one plain read against the single collapsed store. No residency
|
||||
// classification, no legacy probe, no second connection. When the caller passes its
|
||||
// own `_replica` as `newClient`, this is byte-identical to the pre-migration single-DB read.
|
||||
if (deps.splitEnabled === false) {
|
||||
return input.readNew(newClient, runIds);
|
||||
}
|
||||
|
||||
// Split is on. Classify residency; unclassifiable → LEGACY (probe rather than drop).
|
||||
const newIds: string[] = [];
|
||||
const legacyCandidateIds: string[] = [];
|
||||
for (const runId of runIds) {
|
||||
let residency: "LEGACY" | "NEW";
|
||||
try {
|
||||
residency = ownerEngine(runId);
|
||||
} catch (e) {
|
||||
if (e instanceof UnclassifiableRunId) {
|
||||
deps.logger?.warn("hydrateRunsAcrossSeam: UnclassifiableRunId, treating as LEGACY", {
|
||||
runId,
|
||||
valueLength: e.valueLength,
|
||||
});
|
||||
residency = "LEGACY";
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
if (residency === "NEW") {
|
||||
newIds.push(runId);
|
||||
} else {
|
||||
legacyCandidateIds.push(runId);
|
||||
}
|
||||
}
|
||||
|
||||
// Read NEW for everything that could be on new — NEW-residency ids AND legacy-candidates
|
||||
// (read-through is new-FIRST for legacy too) — in one read.
|
||||
const legacyReplica = deps.legacyReplica ?? defaultLegacyReplica;
|
||||
const newRows = await input.readNew(newClient, [...newIds, ...legacyCandidateIds]);
|
||||
const foundOnNew = new Set(newRows.map(getId));
|
||||
|
||||
// Legacy-candidates the new read missed are probed on the legacy read replica.
|
||||
const legacyToProbe = legacyCandidateIds.filter((id) => !foundOnNew.has(id));
|
||||
|
||||
// Legacy READ REPLICA only — never a legacy writer/primary (no such handle exists).
|
||||
// A member absent from both DBs is simply not hydrated (matching today's `findMany`,
|
||||
// where a missing id yields no row).
|
||||
let legacyRows: T[] = [];
|
||||
if (legacyToProbe.length > 0) {
|
||||
legacyRows = await input.readLegacyReplica(legacyReplica, legacyToProbe);
|
||||
}
|
||||
|
||||
// Order within the page is irrelevant (downstream pMap does not depend on it).
|
||||
return [...newRows, ...legacyRows];
|
||||
}
|
||||
@@ -0,0 +1,616 @@
|
||||
import { BulkActionId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
BulkActionNotificationType,
|
||||
BulkActionStatus,
|
||||
BulkActionType,
|
||||
type PrismaClient,
|
||||
} from "@trigger.dev/database";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import {
|
||||
parseRunListInputOptions,
|
||||
type RunListInputFilters,
|
||||
RunsRepository,
|
||||
} from "~/services/runsRepository/runsRepository.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { ServiceValidationError } from "../common.server";
|
||||
import { commonWorker } from "~/v3/commonWorker.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "@trigger.dev/sdk";
|
||||
import { CancelTaskRunService } from "../cancelTaskRun.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { ReplayTaskRunService } from "../replayTaskRun.server";
|
||||
import { WorkerGroupService } from "../worker/workerGroupService.server";
|
||||
import { timeFilters } from "~/components/runs/v3/SharedFilters";
|
||||
import parseDuration from "parse-duration";
|
||||
import { v3BulkActionPath } from "~/utils/pathBuilder";
|
||||
import { formatDateTime } from "~/components/primitives/DateTime";
|
||||
import pMap from "p-map";
|
||||
import { type PrismaReplicaClient } from "~/db.server";
|
||||
import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
|
||||
import { hydrateRunsAcrossSeam, type SeamReadDeps } from "./BulkActionV2.batchReadThrough.server";
|
||||
|
||||
export type CreateBulkActionInput = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
userId?: string | null;
|
||||
action: "cancel" | "replay";
|
||||
filters: RunListInputFilters;
|
||||
title?: string;
|
||||
region?: string;
|
||||
emailNotification?: boolean;
|
||||
triggerSource?: string;
|
||||
};
|
||||
|
||||
export type ProcessToCompletionOptions = {
|
||||
/** Absolute timestamp (ms) after which processing stops and returns incomplete. */
|
||||
deadline?: number;
|
||||
};
|
||||
|
||||
export type ProcessToCompletionResult = {
|
||||
completed: boolean;
|
||||
};
|
||||
|
||||
// How recently a PENDING replay must have made progress to still count against
|
||||
// the per-environment concurrency limit. Every processed batch bumps the
|
||||
// group's `updatedAt`, so a live replay keeps a fresh heartbeat for its whole
|
||||
// life no matter how long it runs, while a replay whose job has exhausted its
|
||||
// retries (and stopped making progress) ages out and frees its slot. This is
|
||||
// wide enough to cover the worst-case gap between batches for a healthy replay
|
||||
// that is retrying.
|
||||
const REPLAY_INFLIGHT_WINDOW_MS = 30 * 60 * 1000;
|
||||
|
||||
export class BulkActionService extends BaseService {
|
||||
#splitEnabledPromise?: Promise<boolean>;
|
||||
|
||||
// Resolves split mode once per service instance and returns the read-through deps for
|
||||
// bulk member hydration. Single-DB: read through the service replica (byte-identical to
|
||||
// the pre-migration read). Split: adapter defaults to run-ops new + legacy read replica.
|
||||
async #seamReadDeps(): Promise<SeamReadDeps> {
|
||||
this.#splitEnabledPromise ??= isSplitEnabled();
|
||||
const splitEnabled = await this.#splitEnabledPromise;
|
||||
return {
|
||||
splitEnabled,
|
||||
newClient: splitEnabled ? undefined : (this._replica as unknown as PrismaReplicaClient),
|
||||
};
|
||||
}
|
||||
|
||||
public async create(input: CreateBulkActionInput) {
|
||||
const { organizationId, projectId, environmentId, userId } = input;
|
||||
const filters = freezeRunListFilters(input.filters);
|
||||
|
||||
// Concurrency guard for replays.
|
||||
// The seek is backed by the (environmentId, status, type) index; the
|
||||
// `updatedAt` window is applied on top so we only count replays that are
|
||||
// actually still making progress. A replay whose job has died stops bumping
|
||||
// `updatedAt` and drops out of the count, so it can't permanently hold a
|
||||
// slot. Aborting a replay (dashboard or API) clears its slot immediately.
|
||||
if (input.action === "replay") {
|
||||
const maxConcurrentReplays = env.BULK_ACTION_MAX_CONCURRENT_REPLAYS;
|
||||
const inFlightReplays = await this._replica.bulkActionGroup.count({
|
||||
where: {
|
||||
environmentId,
|
||||
type: BulkActionType.REPLAY,
|
||||
status: BulkActionStatus.PENDING,
|
||||
updatedAt: { gte: new Date(Date.now() - REPLAY_INFLIGHT_WINDOW_MS) },
|
||||
},
|
||||
});
|
||||
|
||||
if (inFlightReplays >= maxConcurrentReplays) {
|
||||
throw new ServiceValidationError(
|
||||
`You can only run ${maxConcurrentReplays} bulk replays at a time in this environment. Wait for an in-progress replay to finish before starting another.`,
|
||||
429
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Region is a replay-only override that re-routes the replayed runs. It's
|
||||
// stored alongside the run-list filters under a dedicated key so it isn't
|
||||
// mistaken for a `regions` selection filter when the params are parsed.
|
||||
const replayRegion = input.action === "replay" ? input.region : undefined;
|
||||
if (replayRegion) {
|
||||
// Validating the region override up-front so an invalid/unauthorized
|
||||
// region surfaces as a user-input (400) error rather than a 500.
|
||||
const [regionError] = await tryCatch(
|
||||
new WorkerGroupService({ prisma: this._prisma }).getDefaultWorkerGroupForProject({
|
||||
projectId,
|
||||
regionOverride: replayRegion,
|
||||
})
|
||||
);
|
||||
if (regionError) {
|
||||
throw new ServiceValidationError(regionError.message, 400);
|
||||
}
|
||||
}
|
||||
|
||||
const params = {
|
||||
...filters,
|
||||
...(replayRegion ? { replayRegion } : {}),
|
||||
...(input.triggerSource ? { triggerSource: input.triggerSource } : {}),
|
||||
};
|
||||
|
||||
// Count the runs that will be affected by the bulk action
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"standard"
|
||||
);
|
||||
const runsRepository = new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: this._replica as PrismaClient,
|
||||
});
|
||||
const count = await runsRepository.countRuns({
|
||||
organizationId,
|
||||
projectId,
|
||||
environmentId,
|
||||
...filters,
|
||||
});
|
||||
|
||||
// Create the bulk action group
|
||||
const { id, friendlyId } = BulkActionId.generate();
|
||||
const group = await this._prisma.bulkActionGroup.create({
|
||||
data: {
|
||||
id,
|
||||
friendlyId,
|
||||
projectId,
|
||||
environmentId,
|
||||
userId,
|
||||
name: input.title,
|
||||
type: input.action === "cancel" ? BulkActionType.CANCEL : BulkActionType.REPLAY,
|
||||
params,
|
||||
queryName: "bulk_action_v1",
|
||||
totalCount: count,
|
||||
completionNotification:
|
||||
input.emailNotification === true
|
||||
? BulkActionNotificationType.EMAIL
|
||||
: BulkActionNotificationType.NONE,
|
||||
},
|
||||
});
|
||||
|
||||
// Queue the bulk action group for immediate processing
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${group.id}`,
|
||||
job: "processBulkAction",
|
||||
payload: {
|
||||
bulkActionId: group.id,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
bulkActionId: group.friendlyId,
|
||||
};
|
||||
}
|
||||
|
||||
public async processToCompletion(
|
||||
bulkActionId: string,
|
||||
options?: ProcessToCompletionOptions
|
||||
): Promise<ProcessToCompletionResult> {
|
||||
while (true) {
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionId },
|
||||
select: { status: true },
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new Error(`Bulk action group not found: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
return { completed: true };
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
return { completed: false };
|
||||
}
|
||||
|
||||
if (options?.deadline !== undefined && Date.now() >= options.deadline) {
|
||||
return { completed: false };
|
||||
}
|
||||
|
||||
await this.process(bulkActionId, { continueInline: true });
|
||||
}
|
||||
}
|
||||
|
||||
public async process(
|
||||
bulkActionId: string,
|
||||
options?: {
|
||||
continueInline?: boolean;
|
||||
}
|
||||
) {
|
||||
// 1. Get the bulk action group
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionId },
|
||||
select: {
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
projectId: true,
|
||||
environmentId: true,
|
||||
project: {
|
||||
select: {
|
||||
organizationId: true,
|
||||
slug: true,
|
||||
organization: {
|
||||
select: {
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
environment: {
|
||||
select: {
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
type: true,
|
||||
queryName: true,
|
||||
params: true,
|
||||
cursor: true,
|
||||
completionNotification: true,
|
||||
user: {
|
||||
select: {
|
||||
email: true,
|
||||
},
|
||||
},
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new Error(`Bulk action group not found: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (!group.environmentId || !group.environment) {
|
||||
throw new Error(`Bulk action group has no environment: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
logger.log(`Bulk action group already aborted: ${bulkActionId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Parse the params
|
||||
const rawParams = group.params && typeof group.params === "object" ? group.params : {};
|
||||
const finalizeRun = "finalizeRun" in rawParams && (rawParams as any).finalizeRun === true;
|
||||
const replayRegion =
|
||||
"replayRegion" in rawParams && typeof (rawParams as any).replayRegion === "string"
|
||||
? (rawParams as any).replayRegion
|
||||
: undefined;
|
||||
const triggerSource =
|
||||
"triggerSource" in rawParams && typeof (rawParams as any).triggerSource === "string"
|
||||
? (rawParams as any).triggerSource
|
||||
: "dashboard";
|
||||
const filters = parseRunListInputOptions({
|
||||
organizationId: group.project.organizationId,
|
||||
projectId: group.projectId,
|
||||
environmentId: group.environmentId,
|
||||
...rawParams,
|
||||
});
|
||||
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
group.project.organizationId,
|
||||
"standard"
|
||||
);
|
||||
const runsRepository = new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: this._replica as PrismaClient,
|
||||
});
|
||||
|
||||
if (group.queryName !== "bulk_action_v1") {
|
||||
throw new Error(`Bulk action group has invalid query name: ${group.queryName}`);
|
||||
}
|
||||
|
||||
// 2. Get the runs to process in this batch, plus the cursor for the next
|
||||
// batch. The cursor is a composite (created_at, run_id) keyset cursor so the
|
||||
// next batch can't re-include or skip runs.
|
||||
const {
|
||||
runIds: runIdsToProcess,
|
||||
pagination: { nextCursor },
|
||||
} = await runsRepository.listRunIds({
|
||||
...filters,
|
||||
page: {
|
||||
size: env.BULK_ACTION_BATCH_SIZE,
|
||||
cursor:
|
||||
typeof group.cursor === "string" && group.cursor !== null ? group.cursor : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
// 3. Process the runs
|
||||
let successCount = 0;
|
||||
let failureCount = 0;
|
||||
|
||||
switch (group.type) {
|
||||
case BulkActionType.CANCEL: {
|
||||
const cancelService = new CancelTaskRunService(this._prisma);
|
||||
|
||||
const seamDeps = await this.#seamReadDeps();
|
||||
const runs = await hydrateRunsAcrossSeam({
|
||||
runIds: runIdsToProcess,
|
||||
readNew: (client, ids) =>
|
||||
client.taskRun.findMany({
|
||||
where: { id: { in: ids } },
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
friendlyId: true,
|
||||
status: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
taskEventStore: true,
|
||||
},
|
||||
}),
|
||||
readLegacyReplica: (replica, ids) =>
|
||||
replica.taskRun.findMany({
|
||||
where: { id: { in: ids } },
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
friendlyId: true,
|
||||
status: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
taskEventStore: true,
|
||||
},
|
||||
}),
|
||||
deps: seamDeps,
|
||||
});
|
||||
|
||||
await pMap(
|
||||
runs,
|
||||
async (run) => {
|
||||
const [error, result] = await tryCatch(
|
||||
cancelService.call(run, {
|
||||
reason: `Bulk action ${group.friendlyId} cancelled run`,
|
||||
bulkActionId: bulkActionId,
|
||||
finalizeRun,
|
||||
})
|
||||
);
|
||||
if (error) {
|
||||
logger.error("Failed to cancel run", {
|
||||
error,
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
if (!result || result.alreadyFinished) {
|
||||
failureCount++;
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
},
|
||||
{ concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
case BulkActionType.REPLAY: {
|
||||
const replayService = new ReplayTaskRunService(this._prisma);
|
||||
|
||||
const seamDeps = await this.#seamReadDeps();
|
||||
const runs = await hydrateRunsAcrossSeam({
|
||||
runIds: runIdsToProcess,
|
||||
readNew: (client, ids) => client.taskRun.findMany({ where: { id: { in: ids } } }),
|
||||
readLegacyReplica: (replica, ids) =>
|
||||
replica.taskRun.findMany({ where: { id: { in: ids } } }),
|
||||
deps: seamDeps,
|
||||
});
|
||||
|
||||
await pMap(
|
||||
runs,
|
||||
async (run) => {
|
||||
const [error, result] = await tryCatch(
|
||||
replayService.call(run, {
|
||||
bulkActionId: bulkActionId,
|
||||
triggerSource,
|
||||
region: replayRegion,
|
||||
})
|
||||
);
|
||||
if (error) {
|
||||
logger.error("Failed to replay run, error", {
|
||||
error,
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
if (!result) {
|
||||
logger.error("Failed to replay run, no result", {
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
},
|
||||
{ concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A null nextCursor means there is no further page — this batch was the
|
||||
// last (or there were no runs at all), so the action is complete. (An empty
|
||||
// batch also yields a null cursor.)
|
||||
const isFinished = nextCursor === null;
|
||||
|
||||
logger.debug("Bulk action group processed batch", {
|
||||
bulkActionId,
|
||||
organizationId: group.project.organizationId,
|
||||
projectId: group.projectId,
|
||||
environmentId: group.environmentId,
|
||||
batchSize: runIdsToProcess.length,
|
||||
cursor: group.cursor,
|
||||
successCount,
|
||||
failureCount,
|
||||
isFinished,
|
||||
});
|
||||
|
||||
// 4. Update the bulk action group
|
||||
const updatedGroup = await this._prisma.bulkActionGroup.update({
|
||||
where: { id: bulkActionId },
|
||||
data: {
|
||||
// Json column: leave unchanged when there's no next cursor (finished).
|
||||
cursor: nextCursor ?? undefined,
|
||||
successCount: {
|
||||
increment: successCount,
|
||||
},
|
||||
failureCount: {
|
||||
increment: failureCount,
|
||||
},
|
||||
status: isFinished ? BulkActionStatus.COMPLETED : undefined,
|
||||
completedAt: isFinished ? new Date() : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
// 5. If finished, queue a notification and exit
|
||||
if (isFinished) {
|
||||
switch (group.completionNotification) {
|
||||
case BulkActionNotificationType.NONE:
|
||||
return;
|
||||
case BulkActionNotificationType.EMAIL: {
|
||||
if (!group.user) {
|
||||
logger.error("Bulk action group has no user, skipping email notification", {
|
||||
bulkActionId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await commonWorker.enqueue({
|
||||
id: `bulkActionCompletionNotification-${bulkActionId}`,
|
||||
job: "scheduleEmail",
|
||||
payload: {
|
||||
to: group.user.email,
|
||||
email: "bulk-action-completed",
|
||||
bulkActionId: group.friendlyId,
|
||||
url: `${env.LOGIN_ORIGIN}${v3BulkActionPath(
|
||||
{
|
||||
slug: group.project.organization.slug,
|
||||
},
|
||||
{
|
||||
slug: group.project.slug,
|
||||
},
|
||||
{
|
||||
slug: group.environment.slug,
|
||||
},
|
||||
{
|
||||
friendlyId: group.friendlyId,
|
||||
}
|
||||
)}`,
|
||||
totalCount: updatedGroup.totalCount,
|
||||
successCount: updatedGroup.successCount,
|
||||
failureCount: updatedGroup.failureCount,
|
||||
type: group.type,
|
||||
createdAt: formatDateTime(group.createdAt, "UTC", [], true, true),
|
||||
completedAt: formatDateTime(group.completedAt ?? new Date(), "UTC", [], true, true),
|
||||
},
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// 6. If there are more runs to process, queue the next batch
|
||||
if (options?.continueInline) {
|
||||
return;
|
||||
}
|
||||
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${bulkActionId}`,
|
||||
job: "processBulkAction",
|
||||
payload: { bulkActionId },
|
||||
availableAt: new Date(Date.now() + env.BULK_ACTION_BATCH_DELAY_MS),
|
||||
});
|
||||
}
|
||||
|
||||
public async abort(friendlyId: string, environmentId: string) {
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { friendlyId, environmentId },
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new ServiceValidationError(`Bulk action not found: ${friendlyId}`, 404);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
throw new ServiceValidationError(`Bulk action group already completed: ${friendlyId}`, 409);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
throw new ServiceValidationError(`Bulk action group already aborted: ${friendlyId}`, 409);
|
||||
}
|
||||
|
||||
//ack the job (this doesn't guarantee it won't run again)
|
||||
await commonWorker.ack(`processBulkAction-${group.id}`);
|
||||
|
||||
await this._prisma.bulkActionGroup.update({
|
||||
where: { id: group.id },
|
||||
data: { status: BulkActionStatus.ABORTED },
|
||||
});
|
||||
|
||||
return {
|
||||
bulkActionId: friendlyId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function freezeRunListFilters(filters: RunListInputFilters): RunListInputFilters {
|
||||
const {
|
||||
cursor: _cursor,
|
||||
direction: _direction,
|
||||
...frozenFilters
|
||||
} = filters as RunListInputFilters & {
|
||||
cursor?: string;
|
||||
direction?: "forward" | "backward";
|
||||
};
|
||||
|
||||
// Explicit run-id selections target specific, already-existing runs, so we
|
||||
// don't apply a time bound (which could otherwise exclude a selected run).
|
||||
if (frozenFilters.runId?.length) {
|
||||
return frozenFilters;
|
||||
}
|
||||
|
||||
const { period } = timeFilters({
|
||||
period: frozenFilters.period,
|
||||
from: frozenFilters.from,
|
||||
to: frozenFilters.to,
|
||||
});
|
||||
|
||||
// We fix the time period to a from/to date
|
||||
if (period) {
|
||||
const periodMs = parseDuration(period);
|
||||
if (!periodMs) {
|
||||
throw new Error(`Invalid period: ${period}`);
|
||||
}
|
||||
|
||||
const to = new Date();
|
||||
const from = new Date(to.getTime() - periodMs);
|
||||
frozenFilters.from = from.getTime();
|
||||
frozenFilters.to = to.getTime();
|
||||
frozenFilters.period = undefined;
|
||||
return frozenFilters;
|
||||
}
|
||||
|
||||
// If no to date is set, we lock it to now
|
||||
if (!frozenFilters.to) {
|
||||
frozenFilters.to = Date.now();
|
||||
}
|
||||
|
||||
frozenFilters.period = undefined;
|
||||
|
||||
return frozenFilters;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { type BulkActionType } from "@trigger.dev/database";
|
||||
import { bulkActionVerb } from "~/components/runs/v3/BulkAction";
|
||||
import { BULK_ACTION_RUN_LIMIT } from "~/consts";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../../friendlyIdentifiers";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { PerformBulkActionService } from "./performBulkAction.server";
|
||||
|
||||
type BulkAction = {
|
||||
projectId: string;
|
||||
action: BulkActionType;
|
||||
runIds: string[];
|
||||
};
|
||||
|
||||
export class CreateBulkActionService extends BaseService {
|
||||
public async call({ projectId, action, runIds }: BulkAction) {
|
||||
const group = await this._prisma.bulkActionGroup.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("bulk"),
|
||||
projectId,
|
||||
type: action,
|
||||
},
|
||||
});
|
||||
|
||||
//limit to the first X runs
|
||||
const passedTooManyRuns = runIds.length > BULK_ACTION_RUN_LIMIT;
|
||||
runIds = runIds.slice(0, BULK_ACTION_RUN_LIMIT);
|
||||
|
||||
const _items = await this._prisma.bulkActionItem.createMany({
|
||||
data: runIds.map((runId) => ({
|
||||
friendlyId: generateFriendlyId("bulkitem"),
|
||||
type: action,
|
||||
groupId: group.id,
|
||||
sourceRunId: runId,
|
||||
})),
|
||||
});
|
||||
|
||||
logger.debug("Created bulk action group", {
|
||||
groupId: group.id,
|
||||
action,
|
||||
runIds,
|
||||
});
|
||||
|
||||
await PerformBulkActionService.enqueue(group.id, this._prisma);
|
||||
|
||||
let message = bulkActionVerb(action);
|
||||
if (passedTooManyRuns) {
|
||||
message += ` the first ${BULK_ACTION_RUN_LIMIT} runs`;
|
||||
} else {
|
||||
message += ` ${runIds.length} runs`;
|
||||
}
|
||||
|
||||
return {
|
||||
id: group.id,
|
||||
friendlyId: group.friendlyId,
|
||||
runCount: runIds.length,
|
||||
message,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import assertNever from "assert-never";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { CancelTaskRunService } from "../cancelTaskRun.server";
|
||||
import { ReplayTaskRunService } from "../replayTaskRun.server";
|
||||
|
||||
export class PerformBulkActionService extends BaseService {
|
||||
public async performBulkActionItem(bulkActionItemId: string) {
|
||||
const item = await this._prisma.bulkActionItem.findFirst({
|
||||
where: { id: bulkActionItemId },
|
||||
include: {
|
||||
sourceRun: true,
|
||||
destinationRun: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!item) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.status !== "PENDING") {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (item.type) {
|
||||
case "REPLAY": {
|
||||
const service = new ReplayTaskRunService(this._prisma);
|
||||
const result = await service.call(item.sourceRun, { triggerSource: "dashboard" });
|
||||
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
destinationRunId: result?.id,
|
||||
status: result ? "COMPLETED" : "FAILED",
|
||||
error: result ? undefined : "Failed to replay task run",
|
||||
},
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "CANCEL": {
|
||||
const service = new CancelTaskRunService(this._prisma);
|
||||
|
||||
const result = await service.call(item.sourceRun);
|
||||
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
destinationRunId: item.sourceRun.id,
|
||||
status: result ? "COMPLETED" : "FAILED",
|
||||
error: result ? undefined : "Task wasn't cancelable",
|
||||
},
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
assertNever(item.type);
|
||||
}
|
||||
}
|
||||
|
||||
const groupItems = await this._prisma.bulkActionItem.findMany({
|
||||
where: { groupId: item.groupId },
|
||||
select: {
|
||||
status: true,
|
||||
},
|
||||
});
|
||||
|
||||
const isGroupCompleted = groupItems.every((item) => item.status !== "PENDING");
|
||||
|
||||
if (isGroupCompleted) {
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async enqueueBulkActionItem(bulkActionItemId: string, groupId: string) {
|
||||
await workerQueue.enqueue(
|
||||
"v3.performBulkActionItem",
|
||||
{
|
||||
bulkActionItemId,
|
||||
},
|
||||
{
|
||||
jobKey: `performBulkActionItem:${bulkActionItemId}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async call(bulkActionGroupId: string) {
|
||||
const actionGroup = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionGroupId },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
if (!actionGroup) {
|
||||
return;
|
||||
}
|
||||
|
||||
const items = await this._prisma.bulkActionItem.findMany({
|
||||
where: { groupId: bulkActionGroupId },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
for (const item of items) {
|
||||
await this.enqueueBulkActionItem(item.id, bulkActionGroupId);
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(bulkActionGroupId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
|
||||
return await workerQueue.enqueue(
|
||||
"v3.performBulkAction",
|
||||
{
|
||||
bulkActionGroupId,
|
||||
},
|
||||
{
|
||||
tx,
|
||||
runAt,
|
||||
jobKey: `performBulkAction:${bulkActionGroupId}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { $transaction, type PrismaClientOrTransaction, prisma } from "~/db.server";
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { isCancellableRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
|
||||
export class CancelAttemptService extends BaseService {
|
||||
public async call(
|
||||
attemptId: string,
|
||||
taskRunId: string,
|
||||
cancelledAt: Date,
|
||||
reason: string,
|
||||
env?: AuthenticatedEnvironment
|
||||
) {
|
||||
let environment: AuthenticatedEnvironment | undefined = env;
|
||||
|
||||
if (!environment) {
|
||||
environment = await getAuthenticatedEnvironmentFromAttempt(attemptId);
|
||||
|
||||
if (!environment) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
return await this.traceWithEnv("call()", environment, async (span) => {
|
||||
span.setAttribute("taskRunId", taskRunId);
|
||||
span.setAttribute("attemptId", attemptId);
|
||||
|
||||
const taskRunAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: {
|
||||
friendlyId: attemptId,
|
||||
},
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (taskRunAttempt.status === "CANCELED") {
|
||||
logger.warn("Task run attempt is already cancelled", {
|
||||
attemptId,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await $transaction(this._prisma, "cancel attempt", async (tx) => {
|
||||
await tx.taskRunAttempt.update({
|
||||
where: {
|
||||
friendlyId: attemptId,
|
||||
},
|
||||
data: {
|
||||
status: "CANCELED",
|
||||
completedAt: cancelledAt,
|
||||
},
|
||||
});
|
||||
|
||||
const isCancellable = isCancellableRunStatus(taskRunAttempt.taskRun.status);
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService(tx);
|
||||
await finalizeService.call({
|
||||
id: taskRunId,
|
||||
status: isCancellable ? "INTERRUPTED" : undefined,
|
||||
completedAt: isCancellable ? cancelledAt : undefined,
|
||||
attemptStatus: isCancellable ? "CANCELED" : undefined,
|
||||
error: isCancellable ? { type: "STRING_ERROR", raw: reason } : undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function getAuthenticatedEnvironmentFromAttempt(
|
||||
friendlyId: string,
|
||||
prismaClient?: PrismaClientOrTransaction
|
||||
) {
|
||||
const taskRunAttempt = await (prismaClient ?? prisma).taskRunAttempt.findFirst({
|
||||
where: {
|
||||
friendlyId,
|
||||
},
|
||||
include: {
|
||||
runtimeEnvironment: {
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
return;
|
||||
}
|
||||
|
||||
return taskRunAttempt?.runtimeEnvironment;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { type RunStore } from "@internal/run-store";
|
||||
import { z } from "zod";
|
||||
import { type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { findLatestSession } from "~/models/runtimeEnvironment.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { type ReadThroughDeps, readThroughRun } from "../runOpsMigration/readThrough.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { type CancelableTaskRun, CancelTaskRunService } from "./cancelTaskRun.server";
|
||||
|
||||
export const CancelDevSessionRunsServiceOptions = z.object({
|
||||
runIds: z.array(z.string()),
|
||||
cancelledAt: z.coerce.date(),
|
||||
reason: z.string(),
|
||||
cancelledSessionId: z.string().optional(),
|
||||
});
|
||||
|
||||
export type CancelDevSessionRunsServiceOptions = z.infer<typeof CancelDevSessionRunsServiceOptions>;
|
||||
|
||||
export class CancelDevSessionRunsService extends BaseService {
|
||||
// Injectable read-through deps for the run-ops TaskRun read. Undefined in production:
|
||||
// readThroughRun then uses its ~/db.server singleton handles and the boot split flag,
|
||||
// so single-DB is unchanged. Tests inject the hetero new/legacy handles + splitEnabled.
|
||||
readonly #readThroughDeps?: ReadThroughDeps;
|
||||
|
||||
constructor(
|
||||
opts: {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
replica?: PrismaClientOrTransaction;
|
||||
runStore?: RunStore;
|
||||
readThroughDeps?: ReadThroughDeps;
|
||||
} = {}
|
||||
) {
|
||||
super(opts.prisma, opts.replica, opts.runStore);
|
||||
this.#readThroughDeps = opts.readThroughDeps;
|
||||
}
|
||||
|
||||
public async call(options: CancelDevSessionRunsServiceOptions) {
|
||||
const cancelledSession = options.cancelledSessionId
|
||||
? await this._prisma.runtimeEnvironmentSession.findFirst({
|
||||
where: { id: options.cancelledSessionId },
|
||||
})
|
||||
: undefined;
|
||||
|
||||
if (cancelledSession) {
|
||||
const latestSession = await findLatestSession(cancelledSession.environmentId, this._replica);
|
||||
|
||||
if (
|
||||
latestSession &&
|
||||
latestSession.id !== cancelledSession.id &&
|
||||
!latestSession.disconnectedAt
|
||||
) {
|
||||
logger.debug("Not cancelling runs because there is a newer session", {
|
||||
cancelledSessionId: cancelledSession.id,
|
||||
latestSessionId: latestSession.id,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
"Cancelling in progress runs for dev session because there isn't a newer connected session",
|
||||
{
|
||||
options,
|
||||
cancelledSession,
|
||||
}
|
||||
);
|
||||
|
||||
const cancelTaskRunService = new CancelTaskRunService();
|
||||
|
||||
// readThroughRun resolves residency from the run id alone; an env scope is only
|
||||
// available when a cancelled session was resolved.
|
||||
const environmentId = cancelledSession?.environmentId ?? "";
|
||||
|
||||
for (const runId of options.runIds) {
|
||||
await this.#cancelInProgressRun(
|
||||
runId,
|
||||
cancelTaskRunService,
|
||||
options.cancelledAt,
|
||||
options.reason,
|
||||
environmentId
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #cancelInProgressRun(
|
||||
runId: string,
|
||||
service: CancelTaskRunService,
|
||||
cancelledAt: Date,
|
||||
reason: string,
|
||||
environmentId: string
|
||||
) {
|
||||
logger.debug("Cancelling in progress run", { runId });
|
||||
|
||||
// Read-through: new store first, legacy read replica for an old
|
||||
// in-retention run; single plain read in single-DB passthrough.
|
||||
const where = runId.startsWith("run_") ? { friendlyId: runId } : { id: runId };
|
||||
|
||||
const result = await readThroughRun<CancelableTaskRun>({
|
||||
runId,
|
||||
environmentId,
|
||||
readNew: (client) =>
|
||||
client.taskRun.findFirst({
|
||||
where,
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
taskEventStore: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
},
|
||||
}),
|
||||
readLegacy: (replica) =>
|
||||
replica.taskRun.findFirst({
|
||||
where,
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
taskEventStore: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
},
|
||||
}),
|
||||
deps: this.#readThroughDeps,
|
||||
});
|
||||
|
||||
if (result.source === "not-found" || result.source === "past-retention") {
|
||||
return;
|
||||
}
|
||||
|
||||
const taskRun = result.value;
|
||||
|
||||
try {
|
||||
await service.call(taskRun, { reason, cancelAttempts: true, cancelledAt });
|
||||
} catch (e) {
|
||||
logger.error("Failed to cancel in progress run", {
|
||||
runId,
|
||||
error: e,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(options: CancelDevSessionRunsServiceOptions, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
id: options.cancelledSessionId
|
||||
? `cancelDevSessionRuns:${options.cancelledSessionId}`
|
||||
: undefined,
|
||||
job: "v3.cancelDevSessionRuns",
|
||||
payload: options,
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelTaskRunService } from "./cancelTaskRun.server";
|
||||
|
||||
export class CancelTaskAttemptDependenciesService extends BaseService {
|
||||
public async call(attemptId: string) {
|
||||
const taskAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: { id: attemptId },
|
||||
include: {
|
||||
dependencies: {
|
||||
select: {
|
||||
taskRunId: true,
|
||||
},
|
||||
},
|
||||
batchDependencies: {
|
||||
include: {
|
||||
runDependencies: {
|
||||
select: {
|
||||
taskRunId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskAttempt) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (taskAttempt.status !== "CANCELED") {
|
||||
logger.debug("Task attempt is not cancelled, continuing anyway", {
|
||||
attemptId,
|
||||
status: taskAttempt.status,
|
||||
});
|
||||
}
|
||||
|
||||
const cancelRunService = new CancelTaskRunService();
|
||||
|
||||
logger.debug("Cancelling task attempt dependencies", {
|
||||
taskAttempt,
|
||||
dependencies: taskAttempt.dependencies,
|
||||
batchDependencies: taskAttempt.batchDependencies,
|
||||
});
|
||||
|
||||
// Hydrate the dependent runs from both relation paths in a single batched read,
|
||||
// deduping the ids that feed the query while preserving the original iteration order.
|
||||
const taskRunIds = new Set<string>();
|
||||
for (const dependency of taskAttempt.dependencies) {
|
||||
taskRunIds.add(dependency.taskRunId);
|
||||
}
|
||||
for (const batchDependency of taskAttempt.batchDependencies) {
|
||||
for (const runDependency of batchDependency.runDependencies) {
|
||||
taskRunIds.add(runDependency.taskRunId);
|
||||
}
|
||||
}
|
||||
|
||||
const runs =
|
||||
taskRunIds.size > 0
|
||||
? await this.runStore.findRuns(
|
||||
{
|
||||
where: { id: { in: [...taskRunIds] } },
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
taskEventStore: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
)
|
||||
: [];
|
||||
|
||||
const runMap = new Map(runs.map((run) => [run.id, run]));
|
||||
|
||||
// TaskAttempt will either have dependencies or batchDependencies
|
||||
for (const dependency of taskAttempt.dependencies) {
|
||||
const run = runMap.get(dependency.taskRunId);
|
||||
if (run) {
|
||||
await cancelRunService.call(run);
|
||||
}
|
||||
}
|
||||
|
||||
for (const batchDependency of taskAttempt.batchDependencies) {
|
||||
for (const runDependency of batchDependency.runDependencies) {
|
||||
const run = runMap.get(runDependency.taskRunId);
|
||||
if (run) {
|
||||
await cancelRunService.call(run);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(attemptId: string, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
id: `cancelTaskAttemptDependencies:${attemptId}`,
|
||||
job: "v3.cancelTaskAttemptDependencies",
|
||||
payload: {
|
||||
attemptId,
|
||||
},
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { RunEngineVersion, type TaskRun } from "@trigger.dev/database";
|
||||
import { engine } from "../runEngine.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelTaskRunServiceV1 } from "./cancelTaskRunV1.server";
|
||||
|
||||
export type CancelTaskRunServiceOptions = {
|
||||
reason?: string;
|
||||
cancelAttempts?: boolean;
|
||||
cancelledAt?: Date;
|
||||
bulkActionId?: string;
|
||||
/** Skip PENDING_CANCEL and finalize immediately (use when the worker is known to be dead). */
|
||||
finalizeRun?: boolean;
|
||||
};
|
||||
|
||||
type CancelTaskRunServiceResult = {
|
||||
id: string;
|
||||
alreadyFinished: boolean;
|
||||
};
|
||||
|
||||
export type CancelableTaskRun = Pick<
|
||||
TaskRun,
|
||||
"id" | "engine" | "status" | "friendlyId" | "taskEventStore" | "createdAt" | "completedAt"
|
||||
>;
|
||||
|
||||
export class CancelTaskRunService extends BaseService {
|
||||
public async call(
|
||||
taskRun: CancelableTaskRun,
|
||||
options?: CancelTaskRunServiceOptions
|
||||
): Promise<CancelTaskRunServiceResult | undefined> {
|
||||
if (taskRun.engine === RunEngineVersion.V1) {
|
||||
return await this.callV1(taskRun, options);
|
||||
} else {
|
||||
return await this.callV2(taskRun, options);
|
||||
}
|
||||
}
|
||||
|
||||
private async callV1(
|
||||
taskRun: CancelableTaskRun,
|
||||
options?: CancelTaskRunServiceOptions
|
||||
): Promise<CancelTaskRunServiceResult | undefined> {
|
||||
const service = new CancelTaskRunServiceV1(this._prisma);
|
||||
const result = await service.call(taskRun, options);
|
||||
|
||||
if (!result) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
id: result.id,
|
||||
alreadyFinished: false,
|
||||
};
|
||||
}
|
||||
|
||||
private async callV2(
|
||||
taskRun: CancelableTaskRun,
|
||||
options?: CancelTaskRunServiceOptions
|
||||
): Promise<CancelTaskRunServiceResult | undefined> {
|
||||
const result = await engine.cancelRun({
|
||||
runId: taskRun.id,
|
||||
completedAt: options?.cancelledAt,
|
||||
reason: options?.reason,
|
||||
finalizeRun: options?.finalizeRun,
|
||||
bulkActionId: options?.bulkActionId,
|
||||
tx: this._prisma,
|
||||
});
|
||||
|
||||
return {
|
||||
id: result.run.id,
|
||||
alreadyFinished: result.alreadyFinished,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { type Prisma } from "@trigger.dev/database";
|
||||
import assertNever from "assert-never";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { devPubSub } from "../marqs/devPubSub.server";
|
||||
import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelAttemptService } from "./cancelAttempt.server";
|
||||
import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDependencies.server";
|
||||
import type { CancelableTaskRun } from "./cancelTaskRun.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import { getEventRepositoryForStore } from "../eventRepository/index.server";
|
||||
|
||||
type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
|
||||
include: {
|
||||
runtimeEnvironment: true;
|
||||
lockedToVersion: true;
|
||||
};
|
||||
}>;
|
||||
|
||||
type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
|
||||
include: {
|
||||
backgroundWorker: true;
|
||||
};
|
||||
}>;
|
||||
|
||||
export type CancelTaskRunServiceOptions = {
|
||||
reason?: string;
|
||||
cancelAttempts?: boolean;
|
||||
cancelledAt?: Date;
|
||||
bulkActionId?: string;
|
||||
};
|
||||
|
||||
export class CancelTaskRunServiceV1 extends BaseService {
|
||||
public async call(taskRun: CancelableTaskRun, options?: CancelTaskRunServiceOptions) {
|
||||
const opts = {
|
||||
reason: "Task run was cancelled by user",
|
||||
cancelAttempts: true,
|
||||
cancelledAt: new Date(),
|
||||
...options,
|
||||
};
|
||||
|
||||
// Make sure the task run is in a cancellable state
|
||||
if (!isCancellableRunStatus(taskRun.status)) {
|
||||
logger.info("Task run is not in a cancellable state", {
|
||||
runId: taskRun.id,
|
||||
status: taskRun.status,
|
||||
});
|
||||
|
||||
//add the bulk action id to the run
|
||||
if (opts.bulkActionId) {
|
||||
await this._prisma.taskRun.update({
|
||||
where: { id: taskRun.id },
|
||||
data: {
|
||||
bulkActionGroupIds: {
|
||||
push: opts.bulkActionId,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
const cancelledTaskRun = await finalizeService.call({
|
||||
id: taskRun.id,
|
||||
status: "CANCELED",
|
||||
completedAt: opts.cancelledAt,
|
||||
bulkActionId: opts.bulkActionId,
|
||||
include: {
|
||||
attempts: {
|
||||
where: {
|
||||
status: {
|
||||
in: CANCELLABLE_ATTEMPT_STATUSES,
|
||||
},
|
||||
},
|
||||
include: {
|
||||
backgroundWorker: true,
|
||||
dependencies: {
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
},
|
||||
batchTaskRunItems: {
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runtimeEnvironment: true,
|
||||
lockedToVersion: true,
|
||||
project: true,
|
||||
},
|
||||
attemptStatus: "CANCELED",
|
||||
error: {
|
||||
type: "STRING_ERROR",
|
||||
raw: opts.reason,
|
||||
},
|
||||
});
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
cancelledTaskRun.taskEventStore,
|
||||
cancelledTaskRun.runtimeEnvironment.organizationId
|
||||
);
|
||||
|
||||
const [cancelRunEventError] = await tryCatch(
|
||||
eventRepository.cancelRunEvent({
|
||||
reason: opts.reason,
|
||||
run: cancelledTaskRun,
|
||||
cancelledAt: opts.cancelledAt,
|
||||
})
|
||||
);
|
||||
|
||||
if (cancelRunEventError) {
|
||||
logger.error("[CancelTaskRunServiceV1] Failed to cancel run event", {
|
||||
error: cancelRunEventError,
|
||||
runId: cancelledTaskRun.id,
|
||||
});
|
||||
}
|
||||
|
||||
// Cancel any in progress attempts
|
||||
if (opts.cancelAttempts) {
|
||||
await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts);
|
||||
await this.#cancelRemainingRunWorkers(cancelledTaskRun);
|
||||
}
|
||||
|
||||
return {
|
||||
id: cancelledTaskRun.id,
|
||||
};
|
||||
}
|
||||
|
||||
async #cancelPotentiallyRunningAttempts(
|
||||
run: ExtendedTaskRun,
|
||||
attempts: ExtendedTaskRunAttempt[]
|
||||
) {
|
||||
for (const attempt of attempts) {
|
||||
await CancelTaskAttemptDependenciesService.enqueue(attempt.id);
|
||||
|
||||
if (run.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
// Signal the task run attempt to stop
|
||||
await devPubSub.publish(
|
||||
`backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
|
||||
"CANCEL_ATTEMPT",
|
||||
{
|
||||
attemptId: attempt.friendlyId,
|
||||
backgroundWorkerId: attempt.backgroundWorker.friendlyId,
|
||||
taskRunId: run.friendlyId,
|
||||
}
|
||||
);
|
||||
} else {
|
||||
switch (attempt.status) {
|
||||
case "EXECUTING": {
|
||||
// We need to send a cancel message to the coordinator
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", {
|
||||
version: "v1",
|
||||
attemptId: attempt.id,
|
||||
attemptFriendlyId: attempt.friendlyId,
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "PENDING":
|
||||
case "PAUSED": {
|
||||
logger.debug("Cancelling pending or paused attempt", {
|
||||
attempt,
|
||||
});
|
||||
|
||||
const service = new CancelAttemptService();
|
||||
|
||||
await service.call(
|
||||
attempt.friendlyId,
|
||||
run.id,
|
||||
new Date(),
|
||||
"Task run was cancelled by user"
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
case "CANCELED":
|
||||
case "COMPLETED":
|
||||
case "FAILED": {
|
||||
// Do nothing
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
assertNever(attempt.status);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #cancelRemainingRunWorkers(run: ExtendedTaskRun) {
|
||||
if (run.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
// Nothing to do
|
||||
return;
|
||||
}
|
||||
|
||||
// Broadcast cancel message to all coordinators
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
|
||||
version: "v1",
|
||||
runId: run.id,
|
||||
// Give the attempts some time to exit gracefully. If the runs supports lazy attempts, it also supports exit delays.
|
||||
delayInMs: run.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { BackgroundWorkerMetadata, tryCatch } from "@trigger.dev/core/v3";
|
||||
import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction, WorkerDeployment } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { syncTaskIdentifiers } from "~/services/taskIdentifierRegistry.server";
|
||||
import {
|
||||
type TaskMetadataCache,
|
||||
type TaskMetadataEntry,
|
||||
} from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { syncDeclarativeSchedules } from "./createBackgroundWorker.server";
|
||||
import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
|
||||
import { compareDeploymentVersions } from "../utils/deploymentVersions";
|
||||
|
||||
export type ChangeCurrentDeploymentDirection = "promote" | "rollback";
|
||||
|
||||
export class ChangeCurrentDeploymentService extends BaseService {
|
||||
private readonly _taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
prisma?: PrismaClientOrTransaction,
|
||||
replica?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
super(prisma, replica);
|
||||
this._taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
public async call(
|
||||
deployment: WorkerDeployment,
|
||||
direction: ChangeCurrentDeploymentDirection,
|
||||
disableVersionCheck?: boolean
|
||||
) {
|
||||
if (!deployment.workerId) {
|
||||
throw new ServiceValidationError(
|
||||
direction === "promote"
|
||||
? "Deployment is not associated with a worker and cannot be promoted."
|
||||
: "Deployment is not associated with a worker and cannot be rolled back."
|
||||
);
|
||||
}
|
||||
|
||||
if (deployment.status !== "DEPLOYED") {
|
||||
throw new ServiceValidationError(
|
||||
direction === "promote"
|
||||
? "Deployment must be in the DEPLOYED state to be promoted."
|
||||
: "Deployment must be in the DEPLOYED state to be rolled back."
|
||||
);
|
||||
}
|
||||
|
||||
const currentPromotion = await this._prisma.workerDeploymentPromotion.findFirst({
|
||||
where: {
|
||||
environmentId: deployment.environmentId,
|
||||
label: CURRENT_DEPLOYMENT_LABEL,
|
||||
},
|
||||
select: {
|
||||
deployment: {
|
||||
select: { id: true, version: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (currentPromotion) {
|
||||
if (currentPromotion.deployment.id === deployment.id) {
|
||||
throw new ServiceValidationError("Deployment is already the current deployment.");
|
||||
}
|
||||
|
||||
// if there is a current promotion, we have to validate we are moving in the right direction based on the deployment versions
|
||||
if (!disableVersionCheck) {
|
||||
switch (direction) {
|
||||
case "promote": {
|
||||
if (
|
||||
compareDeploymentVersions(currentPromotion.deployment.version, deployment.version) >=
|
||||
0
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot promote a deployment that is older than the current deployment."
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "rollback": {
|
||||
if (
|
||||
compareDeploymentVersions(currentPromotion.deployment.version, deployment.version) <=
|
||||
0
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot rollback to a deployment that is newer than the current deployment."
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//set this deployment as the current deployment for this environment
|
||||
await this._prisma.workerDeploymentPromotion.upsert({
|
||||
where: {
|
||||
environmentId_label: {
|
||||
environmentId: deployment.environmentId,
|
||||
label: CURRENT_DEPLOYMENT_LABEL,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
deploymentId: deployment.id,
|
||||
environmentId: deployment.environmentId,
|
||||
label: CURRENT_DEPLOYMENT_LABEL,
|
||||
},
|
||||
update: {
|
||||
deploymentId: deployment.id,
|
||||
},
|
||||
});
|
||||
|
||||
const [fetchTasksError, tasks] = await tryCatch(
|
||||
this._prisma.backgroundWorkerTask.findMany({
|
||||
where: { workerId: deployment.workerId! },
|
||||
select: {
|
||||
slug: true,
|
||||
triggerSource: true,
|
||||
ttl: true,
|
||||
queue: { select: { id: true, name: true } },
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
if (fetchTasksError) {
|
||||
logger.error("Error fetching worker tasks on deployment change", {
|
||||
error: fetchTasksError,
|
||||
});
|
||||
}
|
||||
|
||||
if (tasks) {
|
||||
// Side effect 1: refresh the `TaskIdentifier` table and the existing
|
||||
// `tids:` Redis cache so the task-listing UI reflects the new deploy.
|
||||
const [syncIdentifiersError] = await tryCatch(
|
||||
syncTaskIdentifiers(
|
||||
deployment.environmentId,
|
||||
deployment.projectId,
|
||||
deployment.workerId!,
|
||||
tasks.map((t) => ({ id: t.slug, triggerSource: t.triggerSource }))
|
||||
)
|
||||
);
|
||||
|
||||
if (syncIdentifiersError) {
|
||||
logger.error("Error syncing task identifiers on deployment change", {
|
||||
error: syncIdentifiersError,
|
||||
});
|
||||
}
|
||||
|
||||
// Side effect 2: refresh the `task-meta:` cache that the queue resolver
|
||||
// reads from. Independent of side effect 1 — if `syncTaskIdentifiers`
|
||||
// throws, the queue resolver still gets a warm cache for the new worker.
|
||||
const metadataEntries: TaskMetadataEntry[] = tasks.map((t) => ({
|
||||
slug: t.slug,
|
||||
ttl: t.ttl,
|
||||
triggerSource: t.triggerSource,
|
||||
queueId: t.queue?.id ?? null,
|
||||
queueName: t.queue?.name ?? "",
|
||||
}));
|
||||
|
||||
// Cache calls log+swallow internally.
|
||||
await this._taskMetaCache.populateByCurrentWorker(
|
||||
deployment.environmentId,
|
||||
deployment.workerId!,
|
||||
metadataEntries
|
||||
);
|
||||
}
|
||||
|
||||
const [scheduleSyncError] = await tryCatch(this.#syncSchedulesForDeployment(deployment));
|
||||
|
||||
if (scheduleSyncError) {
|
||||
logger.error("Error syncing declarative schedules on deployment change", {
|
||||
error: scheduleSyncError,
|
||||
});
|
||||
}
|
||||
|
||||
// Only V1 engine workers need the WAITING_FOR_DEPLOY drain — V2 runs sit
|
||||
// in PENDING_VERSION and are handled out of band, so enqueuing here for V2
|
||||
// just produces empty scans of the TaskRun status index.
|
||||
const worker = await this._prisma.backgroundWorker.findFirst({
|
||||
where: { id: deployment.workerId },
|
||||
select: { engine: true },
|
||||
});
|
||||
|
||||
if (worker?.engine === "V1") {
|
||||
await ExecuteTasksWaitingForDeployService.enqueue(deployment.workerId);
|
||||
}
|
||||
}
|
||||
|
||||
async #syncSchedulesForDeployment(deployment: WorkerDeployment) {
|
||||
const worker = await this._prisma.backgroundWorker.findFirst({
|
||||
where: { id: deployment.workerId! },
|
||||
});
|
||||
|
||||
if (!worker) {
|
||||
logger.error("Worker not found for deployment schedule sync", {
|
||||
deploymentId: deployment.id,
|
||||
workerId: deployment.workerId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const parsed = BackgroundWorkerMetadata.safeParse(worker.metadata);
|
||||
|
||||
if (!parsed.success) {
|
||||
logger.error("Failed to parse worker metadata for schedule sync", {
|
||||
deploymentId: deployment.id,
|
||||
workerId: deployment.workerId,
|
||||
error: parsed.error,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const environment = await this._prisma.runtimeEnvironment.findFirst({
|
||||
where: { id: deployment.environmentId },
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
orgMember: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!environment) {
|
||||
logger.error("Environment not found for deployment schedule sync", {
|
||||
deploymentId: deployment.id,
|
||||
environmentId: deployment.environmentId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await syncDeclarativeSchedules(parsed.data.tasks, worker, environment, this._prisma);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { ZodError } from "zod";
|
||||
import { CronPattern } from "../schedules";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { getLimit } from "~/services/platform.v3.server";
|
||||
import { getTimezones } from "~/utils/timezones.server";
|
||||
import { env } from "~/env.server";
|
||||
import { type PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
|
||||
type Schedule = {
|
||||
cron: string;
|
||||
timezone?: string;
|
||||
taskIdentifier: string;
|
||||
friendlyId?: string;
|
||||
};
|
||||
|
||||
export class CheckScheduleService extends BaseService {
|
||||
public async call(projectId: string, schedule: Schedule, environmentIds: string[]) {
|
||||
//validate the cron expression
|
||||
try {
|
||||
CronPattern.parse(schedule.cron);
|
||||
} catch (e) {
|
||||
if (e instanceof ZodError) {
|
||||
throw new ServiceValidationError(`Invalid cron expression: ${e.issues[0].message}`);
|
||||
}
|
||||
|
||||
throw new ServiceValidationError(
|
||||
`Invalid cron expression: ${e instanceof Error ? e.message : JSON.stringify(e)}`
|
||||
);
|
||||
}
|
||||
|
||||
//chek it's a valid timezone
|
||||
if (schedule.timezone) {
|
||||
const possibleTimezones = getTimezones();
|
||||
if (!possibleTimezones.includes(schedule.timezone)) {
|
||||
throw new ServiceValidationError(
|
||||
`Invalid IANA timezone: '${schedule.timezone}'. View the list of valid timezones at ${env.APP_ORIGIN}/timezones`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
//check the task exists
|
||||
const task = await this._prisma.backgroundWorkerTask.findFirst({
|
||||
where: {
|
||||
slug: schedule.taskIdentifier,
|
||||
projectId: projectId,
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
});
|
||||
|
||||
if (!task) {
|
||||
throw new ServiceValidationError(
|
||||
`Task with identifier ${schedule.taskIdentifier} not found in project.`
|
||||
);
|
||||
}
|
||||
|
||||
if (task.triggerSource !== "SCHEDULED") {
|
||||
throw new ServiceValidationError(
|
||||
`Task with identifier ${schedule.taskIdentifier} is not a scheduled task.`
|
||||
);
|
||||
}
|
||||
|
||||
//check they're within their limit
|
||||
const project = await this._prisma.project.findFirst({
|
||||
where: {
|
||||
id: projectId,
|
||||
},
|
||||
select: {
|
||||
organizationId: true,
|
||||
environments: {
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
archivedAt: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!project) {
|
||||
throw new ServiceValidationError("Project not found");
|
||||
}
|
||||
|
||||
const environments = project.environments.filter((env) => environmentIds.includes(env.id));
|
||||
if (environments.some((env) => env.archivedAt)) {
|
||||
throw new ServiceValidationError("Can't add or edit a schedule for an archived branch");
|
||||
}
|
||||
|
||||
//if creating a schedule, check they're under the limits
|
||||
if (!schedule.friendlyId) {
|
||||
const limit = await getLimit(project.organizationId, "schedules", 100_000_000);
|
||||
const schedulesCount = await CheckScheduleService.getUsedSchedulesCount({
|
||||
prisma: this._prisma,
|
||||
projectId,
|
||||
});
|
||||
|
||||
if (schedulesCount >= limit) {
|
||||
throw new ServiceValidationError(
|
||||
`You have created ${schedulesCount}/${limit} schedules so you'll need to increase your limits or delete some schedules.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async getUsedSchedulesCount({
|
||||
prisma,
|
||||
projectId,
|
||||
}: {
|
||||
prisma: PrismaClientOrTransaction;
|
||||
projectId: string;
|
||||
}) {
|
||||
return await prisma.taskScheduleInstance.count({
|
||||
where: {
|
||||
projectId,
|
||||
active: true,
|
||||
environment: {
|
||||
type: {
|
||||
not: "DEVELOPMENT",
|
||||
},
|
||||
archivedAt: null,
|
||||
},
|
||||
taskSchedule: {
|
||||
active: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import {
|
||||
type PendingVersionRunIdLookup,
|
||||
type PendingVersionRunIdLookupOptions,
|
||||
type PendingVersionRunIdLookupResult,
|
||||
} from "@internal/run-engine";
|
||||
import type { Logger } from "@trigger.dev/core/logger";
|
||||
import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
|
||||
|
||||
export type ClickhousePendingVersionLookupOptions = {
|
||||
clickhouseFactory: ClickhouseFactory;
|
||||
logger: Logger;
|
||||
};
|
||||
|
||||
/**
|
||||
* ClickHouse-backed lookup for `PENDING_VERSION` TaskRun ids.
|
||||
*
|
||||
* Resolves the ClickHouse client per call via the
|
||||
* {@link ClickhouseFactory}, which honors per-organization data-store
|
||||
* routing (HIPAA / data-sovereignty customers get their own instance,
|
||||
* everyone else lands on the shared `engine` client configured by
|
||||
* `RUN_ENGINE_CLICKHOUSE_*` env vars).
|
||||
*
|
||||
* Best-effort by design: replication lag against `task_runs_v2` can
|
||||
* produce stale candidates. The run-engine consumer re-validates every
|
||||
* id against Postgres by primary key with a `status = 'PENDING_VERSION'`
|
||||
* guard before any mutation, so stale ids are dropped at the source of
|
||||
* truth. On ClickHouse error we log and return an empty result; the
|
||||
* pending-version re-enqueue tail loop retries on the next event.
|
||||
*/
|
||||
export class ClickhousePendingVersionLookup implements PendingVersionRunIdLookup {
|
||||
readonly name = "clickhouse";
|
||||
|
||||
constructor(private readonly opts: ClickhousePendingVersionLookupOptions) {}
|
||||
|
||||
async lookupPendingVersionRunIds(
|
||||
options: PendingVersionRunIdLookupOptions
|
||||
): Promise<PendingVersionRunIdLookupResult> {
|
||||
// Empty IN-lists would be a no-op; bail before issuing the query.
|
||||
if (options.taskIdentifiers.length === 0 || options.queues.length === 0) {
|
||||
return { runIds: [] };
|
||||
}
|
||||
|
||||
let clickhouse;
|
||||
try {
|
||||
clickhouse = await this.opts.clickhouseFactory.getClickhouseForOrganization(
|
||||
options.organizationId,
|
||||
"engine"
|
||||
);
|
||||
} catch (error) {
|
||||
// Factory resolution failures usually mean a real configuration
|
||||
// problem (registry misload, missing data store, ClientType mismatch).
|
||||
// These are not transient — log at error so ops sees them in dashboards
|
||||
// and incident hooks. Query-level errors below stay at warn because
|
||||
// those are expected to be transient.
|
||||
this.opts.logger.error("ClickhousePendingVersionLookup factory resolution failed", {
|
||||
error,
|
||||
organizationId: options.organizationId,
|
||||
});
|
||||
return { runIds: [] };
|
||||
}
|
||||
|
||||
const builder = clickhouse.taskRuns
|
||||
.pendingVersionIdsQueryBuilder()
|
||||
// `organization_id` MUST be the leading filter — it is the leading
|
||||
// sort-key column on `task_runs_v2` and the only thing that prunes
|
||||
// granules cheaply on a multi-tenant table.
|
||||
.where("organization_id = {organizationId: String}", {
|
||||
organizationId: options.organizationId,
|
||||
})
|
||||
.where("project_id = {projectId: String}", { projectId: options.projectId })
|
||||
.where("environment_id = {environmentId: String}", {
|
||||
environmentId: options.environmentId,
|
||||
})
|
||||
.where("status = 'PENDING_VERSION'")
|
||||
.where("task_identifier IN {taskIdentifiers: Array(String)}", {
|
||||
taskIdentifiers: options.taskIdentifiers,
|
||||
})
|
||||
.where("queue IN {queues: Array(String)}", { queues: options.queues })
|
||||
.where("_is_deleted = 0")
|
||||
.orderBy("created_at ASC")
|
||||
.limit(options.limit);
|
||||
|
||||
const [queryError, rows] = await builder.execute();
|
||||
|
||||
if (queryError) {
|
||||
this.opts.logger.warn("ClickhousePendingVersionLookup query failed", {
|
||||
error: queryError,
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
});
|
||||
return { runIds: [] };
|
||||
}
|
||||
|
||||
return { runIds: rows.map((row) => row.run_id) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
export type ServiceValidationErrorLevel = "error" | "warn" | "info";
|
||||
|
||||
export class ServiceValidationError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public status?: number,
|
||||
public logLevel?: ServiceValidationErrorLevel
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ServiceValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown when a trigger is rejected because the environment's queue is at its
|
||||
* maximum size. This is identified separately from other validation errors so
|
||||
* the batch queue worker can short-circuit retries and skip pre-failed run
|
||||
* creation for this specific overload scenario — see the batch process item
|
||||
* callback in `runEngineHandlers.server.ts`.
|
||||
*/
|
||||
export class QueueSizeLimitExceededError extends ServiceValidationError {
|
||||
constructor(
|
||||
message: string,
|
||||
public maximumSize: number,
|
||||
status?: number,
|
||||
logLevel?: ServiceValidationErrorLevel
|
||||
) {
|
||||
super(message, status, logLevel);
|
||||
this.name = "QueueSizeLimitExceededError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,728 @@
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import type {
|
||||
MachinePresetName,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionResult,
|
||||
TaskRunExecutionRetry,
|
||||
TaskRunFailedExecutionResult,
|
||||
TaskRunSuccessfulExecutionResult,
|
||||
V3TaskRunExecution,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import {
|
||||
TaskRunContext,
|
||||
TaskRunErrorCodes,
|
||||
flattenAttributes,
|
||||
isOOMRunError,
|
||||
sanitizeError,
|
||||
shouldRetryError,
|
||||
taskRunErrorEnhancer,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { createExceptionPropertiesFromError } from "../eventRepository/common.server";
|
||||
import type { FAILED_RUN_STATUSES } from "../taskStatus";
|
||||
import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelAttemptService } from "./cancelAttempt.server";
|
||||
import { CreateCheckpointService } from "./createCheckpoint.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
import { RetryAttemptService } from "./retryAttempt.server";
|
||||
import { getEventRepositoryForStore } from "../eventRepository/index.server";
|
||||
|
||||
type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
|
||||
|
||||
type CheckpointData = {
|
||||
docker: boolean;
|
||||
location: string;
|
||||
};
|
||||
|
||||
type CompleteAttemptServiceOptions = {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
supportsRetryCheckpoints?: boolean;
|
||||
isSystemFailure?: boolean;
|
||||
isCrash?: boolean;
|
||||
};
|
||||
|
||||
export class CompleteAttemptService extends BaseService {
|
||||
constructor(private opts: CompleteAttemptServiceOptions = {}) {
|
||||
super(opts.prisma);
|
||||
}
|
||||
|
||||
public async call({
|
||||
completion,
|
||||
execution,
|
||||
env,
|
||||
checkpoint,
|
||||
}: {
|
||||
completion: TaskRunExecutionResult;
|
||||
execution: V3TaskRunExecution;
|
||||
env?: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
}): Promise<"COMPLETED" | "RETRIED"> {
|
||||
const taskRunAttempt = await findAttempt(this._prisma, execution.attempt.id);
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
logger.error("[CompleteAttemptService] Task run attempt not found", {
|
||||
id: execution.attempt.id,
|
||||
});
|
||||
|
||||
const run = await this.runStore.findRun(
|
||||
{
|
||||
friendlyId: execution.run.id,
|
||||
},
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
logger.error("[CompleteAttemptService] Task run not found", {
|
||||
friendlyId: execution.run.id,
|
||||
});
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: run.id,
|
||||
status: "SYSTEM_FAILURE",
|
||||
completedAt: new Date(),
|
||||
attemptStatus: "FAILED",
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
|
||||
message: "Tried to complete attempt but it doesn't exist",
|
||||
},
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
// No attempt, so there's no message to ACK
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
if (
|
||||
isFinalAttemptStatus(taskRunAttempt.status) ||
|
||||
isFinalRunStatus(taskRunAttempt.taskRun.status)
|
||||
) {
|
||||
// We don't want to retry a task run that has already been marked as failed, cancelled, or completed
|
||||
logger.debug("[CompleteAttemptService] Attempt or run is already in a final state", {
|
||||
taskRunAttempt,
|
||||
completion,
|
||||
});
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
if (completion.ok) {
|
||||
return await this.#completeAttemptSuccessfully(completion, taskRunAttempt, env);
|
||||
} else {
|
||||
return await this.#completeAttemptFailed({
|
||||
completion,
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
env,
|
||||
checkpoint,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async #completeAttemptSuccessfully(
|
||||
completion: TaskRunSuccessfulExecutionResult,
|
||||
taskRunAttempt: NonNullable<FoundAttempt>,
|
||||
env?: AuthenticatedEnvironment
|
||||
): Promise<"COMPLETED"> {
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: { id: taskRunAttempt.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
completedAt: new Date(),
|
||||
output: completion.output,
|
||||
outputType: completion.outputType,
|
||||
usageDurationMs: completion.usage?.durationMs,
|
||||
taskRun: {
|
||||
update: {
|
||||
output: completion.output,
|
||||
outputType: completion.outputType,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status: "COMPLETED_SUCCESSFULLY",
|
||||
completedAt: new Date(),
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
const [completeSuccessfulRunEventError] = await tryCatch(
|
||||
eventRepository.completeSuccessfulRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: new Date(),
|
||||
})
|
||||
);
|
||||
|
||||
if (completeSuccessfulRunEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to complete successful run event", {
|
||||
error: completeSuccessfulRunEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
async #completeAttemptFailed({
|
||||
completion,
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
env,
|
||||
checkpoint,
|
||||
}: {
|
||||
completion: TaskRunFailedExecutionResult;
|
||||
execution: V3TaskRunExecution;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
env?: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
}): Promise<"COMPLETED" | "RETRIED"> {
|
||||
if (
|
||||
completion.error.type === "INTERNAL_ERROR" &&
|
||||
completion.error.code === "TASK_RUN_CANCELLED"
|
||||
) {
|
||||
// We need to cancel the task run instead of fail it
|
||||
const cancelService = new CancelAttemptService();
|
||||
|
||||
// TODO: handle usages
|
||||
await cancelService.call(
|
||||
taskRunAttempt.friendlyId,
|
||||
taskRunAttempt.taskRunId,
|
||||
new Date(),
|
||||
"Canceled by user",
|
||||
env
|
||||
);
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
const failedAt = new Date();
|
||||
const sanitizedError = sanitizeError(completion.error);
|
||||
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: { id: taskRunAttempt.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
completedAt: failedAt,
|
||||
error: sanitizedError,
|
||||
usageDurationMs: completion.usage?.durationMs,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = env ?? (await this.#getEnvironment(execution.environment.id));
|
||||
|
||||
// This means that tasks won't know they are being retried
|
||||
let executionRetryInferred = false;
|
||||
let executionRetry = completion.retry;
|
||||
|
||||
const shouldInfer = this.opts.isCrash || this.opts.isSystemFailure;
|
||||
|
||||
if (!executionRetry && shouldInfer) {
|
||||
executionRetryInferred = true;
|
||||
executionRetry = FailedTaskRunRetryHelper.getExecutionRetry({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
}
|
||||
|
||||
let retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error));
|
||||
let isOOMRetry = false;
|
||||
let isOOMAttempt = isOOMRunError(completion.error);
|
||||
let isOnMaxOOMMachine = false;
|
||||
let oomMachine: MachinePresetName | undefined;
|
||||
|
||||
//OOM errors should retry (if an OOM machine is specified, and we're not already on it)
|
||||
if (isOOMAttempt) {
|
||||
const retryConfig = FailedTaskRunRetryHelper.getRetryConfig({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
|
||||
oomMachine = retryConfig?.outOfMemory?.machine;
|
||||
isOnMaxOOMMachine = oomMachine === taskRunAttempt.taskRun.machinePreset;
|
||||
|
||||
if (oomMachine && !isOnMaxOOMMachine) {
|
||||
//we will retry
|
||||
isOOMRetry = true;
|
||||
retriableError = true;
|
||||
executionRetry = FailedTaskRunRetryHelper.getExecutionRetry({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
|
||||
//update the machine on the run
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
machinePreset: oomMachine,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
retriableError &&
|
||||
executionRetry !== undefined &&
|
||||
taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS
|
||||
) {
|
||||
return await this.#retryAttempt({
|
||||
execution,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
taskRunAttempt,
|
||||
environment,
|
||||
checkpoint,
|
||||
forceRequeue: isOOMRetry,
|
||||
oomMachine,
|
||||
});
|
||||
}
|
||||
|
||||
// The attempt has failed and we won't retry
|
||||
|
||||
if (isOOMAttempt && isOnMaxOOMMachine && environment.type !== "DEVELOPMENT") {
|
||||
// The attempt failed due to an OOM error but we're already on the machine we should retry on
|
||||
exitRun(taskRunAttempt.taskRunId);
|
||||
}
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
const [completeFailedRunEventError] = await tryCatch(
|
||||
eventRepository.completeFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (completeFailedRunEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to complete failed run event", {
|
||||
error: completeFailedRunEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
error: sanitizedError,
|
||||
},
|
||||
});
|
||||
|
||||
let status: FAILED_RUN_STATUSES;
|
||||
|
||||
// Set the correct task run status
|
||||
if (this.opts.isSystemFailure) {
|
||||
status = "SYSTEM_FAILURE";
|
||||
} else if (this.opts.isCrash) {
|
||||
status = "CRASHED";
|
||||
} else if (
|
||||
sanitizedError.type === "INTERNAL_ERROR" &&
|
||||
sanitizedError.code === "MAX_DURATION_EXCEEDED"
|
||||
) {
|
||||
status = "TIMED_OUT";
|
||||
} else if (sanitizedError.type === "INTERNAL_ERROR") {
|
||||
status = "CRASHED";
|
||||
} else {
|
||||
status = "COMPLETED_WITH_ERRORS";
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status,
|
||||
completedAt: failedAt,
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
if (status !== "CRASHED" && status !== "SYSTEM_FAILURE") {
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
// Handle in-progress events
|
||||
switch (status) {
|
||||
case "CRASHED": {
|
||||
const [createAttemptFailedEventError] = await tryCatch(
|
||||
eventRepository.createAttemptFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
attemptNumber: taskRunAttempt.number,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (createAttemptFailedEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
|
||||
error: createAttemptFailedEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "SYSTEM_FAILURE": {
|
||||
const [createAttemptFailedEventError] = await tryCatch(
|
||||
eventRepository.createAttemptFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
attemptNumber: taskRunAttempt.number,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (createAttemptFailedEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
|
||||
error: createAttemptFailedEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
async #enqueueReattempt({
|
||||
run,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpointEventId,
|
||||
supportsLazyAttempts,
|
||||
forceRequeue = false,
|
||||
}: {
|
||||
run: TaskRun;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
checkpointEventId?: string;
|
||||
supportsLazyAttempts: boolean;
|
||||
forceRequeue?: boolean;
|
||||
}) {
|
||||
const retryViaQueue = () => {
|
||||
logger.debug("[CompleteAttemptService] Enqueuing retry attempt", { runId: run.id });
|
||||
|
||||
return marqs.requeueMessage(
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
checkpointEventId: this.opts.supportsRetryCheckpoints ? checkpointEventId : undefined,
|
||||
retryCheckpointsDisabled: !this.opts.supportsRetryCheckpoints,
|
||||
},
|
||||
executionRetry.timestamp,
|
||||
"retry"
|
||||
);
|
||||
};
|
||||
|
||||
const retryDirectly = () => {
|
||||
logger.debug("[CompleteAttemptService] Retrying attempt directly", { runId: run.id });
|
||||
return RetryAttemptService.enqueue(run.id, new Date(executionRetry.timestamp));
|
||||
};
|
||||
|
||||
// There's a checkpoint, so we need to go through the queue
|
||||
if (checkpointEventId) {
|
||||
if (!this.opts.supportsRetryCheckpoints) {
|
||||
logger.error(
|
||||
"[CompleteAttemptService] Worker does not support retry checkpoints, but a checkpoint was created",
|
||||
{
|
||||
runId: run.id,
|
||||
checkpointEventId,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
logger.debug("[CompleteAttemptService] Enqueuing retry attempt with checkpoint", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
|
||||
if (!supportsLazyAttempts) {
|
||||
logger.debug("[CompleteAttemptService] Worker does not support lazy attempts", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
if (forceRequeue) {
|
||||
logger.debug("[CompleteAttemptService] Forcing retry via queue", { runId: run.id });
|
||||
|
||||
// The run won't know it should shut down as we make the decision to force requeue here
|
||||
// This also ensures that this change is backwards compatible with older workers
|
||||
exitRun(run.id);
|
||||
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// Workers that never checkpoint between attempts will exit after completing their current attempt if the retry delay exceeds the threshold
|
||||
if (
|
||||
!this.opts.supportsRetryCheckpoints &&
|
||||
executionRetry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS
|
||||
) {
|
||||
logger.debug(
|
||||
"[CompleteAttemptService] Worker does not support retry checkpoints and the delay exceeds the threshold",
|
||||
{ runId: run.id }
|
||||
);
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
if (executionRetryInferred) {
|
||||
logger.debug("[CompleteAttemptService] Execution retry inferred, forcing retry via queue", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// The worker is still running and waiting for a retry message
|
||||
await retryDirectly();
|
||||
}
|
||||
|
||||
async #retryAttempt({
|
||||
execution,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
taskRunAttempt,
|
||||
environment,
|
||||
checkpoint,
|
||||
forceRequeue = false,
|
||||
oomMachine,
|
||||
}: {
|
||||
execution: V3TaskRunExecution;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
environment: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
forceRequeue?: boolean;
|
||||
/** Setting this will also alter the retry span message */
|
||||
oomMachine?: MachinePresetName;
|
||||
}) {
|
||||
const retryAt = new Date(executionRetry.timestamp);
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
// Retry the task run
|
||||
await eventRepository.recordEvent(
|
||||
`Retry #${execution.attempt.number} delay${oomMachine ? " after OOM" : ""}`,
|
||||
{
|
||||
taskSlug: taskRunAttempt.taskRun.taskIdentifier,
|
||||
environment,
|
||||
attributes: {
|
||||
metadata: this.#generateMetadataAttributesForNextAttempt(execution),
|
||||
properties: {
|
||||
retryAt: retryAt.toISOString(),
|
||||
previousMachine: oomMachine
|
||||
? (taskRunAttempt.taskRun.machinePreset ?? undefined)
|
||||
: undefined,
|
||||
nextMachine: oomMachine,
|
||||
},
|
||||
runId: taskRunAttempt.taskRun.friendlyId,
|
||||
style: {
|
||||
icon: "schedule-attempt",
|
||||
},
|
||||
},
|
||||
context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
|
||||
spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
|
||||
endTime: retryAt,
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug("[CompleteAttemptService] Retrying", {
|
||||
taskRun: taskRunAttempt.taskRun.friendlyId,
|
||||
retry: executionRetry,
|
||||
});
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
status: "RETRYING_AFTER_FAILURE",
|
||||
},
|
||||
});
|
||||
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
await marqs.requeueMessage(taskRunAttempt.taskRunId, {}, executionRetry.timestamp, "retry");
|
||||
|
||||
return "RETRIED";
|
||||
}
|
||||
|
||||
if (checkpoint) {
|
||||
// This is only here for backwards compat - we don't checkpoint between attempts anymore
|
||||
return await this.#retryAttemptWithCheckpoint({
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpoint,
|
||||
});
|
||||
}
|
||||
|
||||
await this.#enqueueReattempt({
|
||||
run: taskRunAttempt.taskRun,
|
||||
executionRetry,
|
||||
supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
|
||||
executionRetryInferred,
|
||||
forceRequeue,
|
||||
});
|
||||
|
||||
return "RETRIED";
|
||||
}
|
||||
|
||||
async #retryAttemptWithCheckpoint({
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpoint,
|
||||
}: {
|
||||
execution: V3TaskRunExecution;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
checkpoint: CheckpointData;
|
||||
}) {
|
||||
const createCheckpoint = new CreateCheckpointService(this._prisma);
|
||||
const checkpointCreateResult = await createCheckpoint.call({
|
||||
attemptFriendlyId: execution.attempt.id,
|
||||
docker: checkpoint.docker,
|
||||
location: checkpoint.location,
|
||||
reason: {
|
||||
type: "RETRYING_AFTER_FAILURE",
|
||||
attemptNumber: execution.attempt.number,
|
||||
},
|
||||
});
|
||||
|
||||
if (!checkpointCreateResult.success) {
|
||||
logger.error("[CompleteAttemptService] Failed to create reattempt checkpoint", {
|
||||
checkpoint,
|
||||
runId: execution.run.id,
|
||||
attemptId: execution.attempt.id,
|
||||
});
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status: "SYSTEM_FAILURE",
|
||||
completedAt: new Date(),
|
||||
error: {
|
||||
type: "STRING_ERROR",
|
||||
raw: "Failed to create reattempt checkpoint",
|
||||
},
|
||||
});
|
||||
|
||||
return "COMPLETED" as const;
|
||||
}
|
||||
|
||||
await this.#enqueueReattempt({
|
||||
run: taskRunAttempt.taskRun,
|
||||
executionRetry,
|
||||
checkpointEventId: checkpointCreateResult.event.id,
|
||||
supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
|
||||
executionRetryInferred,
|
||||
});
|
||||
|
||||
return "RETRIED" as const;
|
||||
}
|
||||
|
||||
#generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {
|
||||
const context = TaskRunContext.parse(execution);
|
||||
|
||||
// @ts-ignore
|
||||
context.attempt = {
|
||||
number: context.attempt.number + 1,
|
||||
};
|
||||
|
||||
return flattenAttributes(context, "ctx");
|
||||
}
|
||||
|
||||
async #getEnvironment(id: string) {
|
||||
return await this._prisma.runtimeEnvironment.findFirstOrThrow({
|
||||
where: {
|
||||
id,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId: string) {
|
||||
return prismaClient.taskRunAttempt.findFirst({
|
||||
where: { friendlyId },
|
||||
include: {
|
||||
taskRun: true,
|
||||
backgroundWorkerTask: true,
|
||||
backgroundWorker: {
|
||||
select: {
|
||||
id: true,
|
||||
supportsLazyAttempts: true,
|
||||
sdkVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function exitRun(runId: string) {
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
|
||||
version: "v1",
|
||||
runId,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
import { ComputeClient, stripImageDigest } from "@internal/compute";
|
||||
import type { TemplateCreateResultEntry } from "@internal/compute";
|
||||
import type { MachinePresetName } from "@trigger.dev/core/v3";
|
||||
import { machinePresetFromName } from "~/v3/machinePresets.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { ServiceValidationError } from "./baseService.server";
|
||||
import { FailDeploymentService } from "./failDeployment.server";
|
||||
import { resolveComputeAccess } from "../regionAccess.server";
|
||||
import { isOrgMigrated } from "~/runEngine/concerns/computeMigration.server";
|
||||
import { backingForQueue, workerRegionRegistry } from "~/v3/workerRegions.server";
|
||||
import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
|
||||
import { getEntitlement } from "~/services/platform.v3.server";
|
||||
import { startActiveSpan, attributesFromAuthenticatedEnv } from "~/v3/tracer.server";
|
||||
|
||||
type TemplateCreationMode = "required" | "shadow" | "skip";
|
||||
|
||||
// Why the mode was chosen — slices the compute.template.create span by path.
|
||||
type TemplateModeReason =
|
||||
| "no-client"
|
||||
| "no-project"
|
||||
| "microvm-native"
|
||||
| "migrated"
|
||||
| "compute-access"
|
||||
| "rollout"
|
||||
| "none";
|
||||
|
||||
type ResolvedTemplateMode = {
|
||||
mode: TemplateCreationMode;
|
||||
migrated: boolean;
|
||||
reason: TemplateModeReason;
|
||||
};
|
||||
|
||||
type ResolvedPreset = {
|
||||
name: MachinePresetName;
|
||||
cpu: number;
|
||||
memory_gb: number;
|
||||
};
|
||||
|
||||
export class ComputeTemplateCreationService {
|
||||
private client: ComputeClient | undefined;
|
||||
private presets: ResolvedPreset[];
|
||||
private requiredPresets: Set<MachinePresetName>;
|
||||
|
||||
constructor() {
|
||||
if (env.COMPUTE_GATEWAY_URL) {
|
||||
this.client = new ComputeClient({
|
||||
gatewayUrl: env.COMPUTE_GATEWAY_URL,
|
||||
authToken: env.COMPUTE_GATEWAY_AUTH_TOKEN,
|
||||
timeoutMs: 5 * 60 * 1000, // 5 minutes
|
||||
});
|
||||
}
|
||||
|
||||
this.presets = env.COMPUTE_TEMPLATE_MACHINE_PRESETS.map((name) => {
|
||||
const machine = machinePresetFromName(name);
|
||||
return { name, cpu: machine.cpu, memory_gb: machine.memory };
|
||||
});
|
||||
this.requiredPresets = new Set(env.COMPUTE_TEMPLATE_MACHINE_PRESETS_REQUIRED);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle template creation for a deployment. Call this before setting DEPLOYED.
|
||||
*
|
||||
* - Required mode: creates template synchronously, fails deployment on error
|
||||
* - Shadow mode: fires background template creation (returns immediately)
|
||||
* - Skip: no-op
|
||||
*
|
||||
* Throws ServiceValidationError if required mode fails (caller should stop finalize).
|
||||
*/
|
||||
async handleDeployTemplate(options: {
|
||||
projectId: string;
|
||||
imageReference: string;
|
||||
deploymentFriendlyId: string;
|
||||
authenticatedEnv: AuthenticatedEnvironment;
|
||||
prisma: PrismaClientOrTransaction;
|
||||
writer?: WritableStreamDefaultWriter;
|
||||
}): Promise<void> {
|
||||
return startActiveSpan("compute.template.create", async (span) => {
|
||||
const { mode, migrated, reason } = await this.resolveMode(options.projectId, options.prisma);
|
||||
|
||||
span.setAttributes({
|
||||
...attributesFromAuthenticatedEnv(options.authenticatedEnv),
|
||||
"compute.template.mode": mode,
|
||||
"compute.template.migrated": migrated,
|
||||
"compute.template.reason": reason,
|
||||
"compute.template.deployment_id": options.deploymentFriendlyId,
|
||||
"compute.template.presets_total": this.presets.length,
|
||||
"compute.template.presets_required": this.requiredPresets.size,
|
||||
});
|
||||
|
||||
if (mode === "skip") {
|
||||
span.setAttribute("compute.template.result", "skipped");
|
||||
return;
|
||||
}
|
||||
|
||||
if (mode === "shadow") {
|
||||
// Shadow is fire-and-forget (background build), so the span only records
|
||||
// that it was dispatched — the build outcome lands server-side later.
|
||||
span.setAttribute("compute.template.result", "shadow_dispatched");
|
||||
this.createTemplate(options.imageReference, { background: true })
|
||||
.then((outcome) => {
|
||||
if (outcome.error) {
|
||||
logger.error("Shadow template creation failed", {
|
||||
id: options.deploymentFriendlyId,
|
||||
imageReference: options.imageReference,
|
||||
error: outcome.error,
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error("Shadow template creation threw unexpectedly", {
|
||||
id: options.deploymentFriendlyId,
|
||||
imageReference: options.imageReference,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Required mode
|
||||
if (options.writer) {
|
||||
try {
|
||||
await options.writer.write(
|
||||
`event: log\ndata: ${JSON.stringify({ message: "Building compute template..." })}\n\n`
|
||||
);
|
||||
} catch {
|
||||
// Stream may be closed if client disconnected - continue with template creation
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Creating compute template (required mode)", {
|
||||
id: options.deploymentFriendlyId,
|
||||
imageReference: options.imageReference,
|
||||
presets: this.presets.map((p) => p.name),
|
||||
requiredPresets: [...this.requiredPresets],
|
||||
});
|
||||
|
||||
const outcome = await this.createTemplate(options.imageReference);
|
||||
span.setAttribute("compute.template.presets_built", outcome.results.length);
|
||||
|
||||
const failureMessage = this.failureMessageForRequiredMode(
|
||||
outcome,
|
||||
options.deploymentFriendlyId,
|
||||
options.imageReference
|
||||
);
|
||||
|
||||
if (failureMessage) {
|
||||
span.setAttributes({
|
||||
"compute.template.result": "failed",
|
||||
"compute.template.failure": failureMessage,
|
||||
});
|
||||
|
||||
logger.error("Compute template creation failed", {
|
||||
id: options.deploymentFriendlyId,
|
||||
imageReference: options.imageReference,
|
||||
error: failureMessage,
|
||||
});
|
||||
|
||||
const failService = new FailDeploymentService();
|
||||
await failService.call(options.authenticatedEnv, options.deploymentFriendlyId, {
|
||||
error: {
|
||||
name: "TemplateCreationFailed",
|
||||
message: `Failed to create compute template: ${failureMessage}`,
|
||||
},
|
||||
});
|
||||
|
||||
throw new ServiceValidationError(`Compute template creation failed: ${failureMessage}`);
|
||||
}
|
||||
|
||||
span.setAttribute("compute.template.result", "created");
|
||||
logger.info("Compute template created", {
|
||||
id: options.deploymentFriendlyId,
|
||||
imageReference: options.imageReference,
|
||||
results: outcome.results.length,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async resolveMode(
|
||||
projectId: string,
|
||||
prisma: PrismaClientOrTransaction
|
||||
): Promise<ResolvedTemplateMode> {
|
||||
if (!this.client) {
|
||||
return { mode: "skip", migrated: false, reason: "no-client" };
|
||||
}
|
||||
|
||||
const project = await prisma.project.findFirst({
|
||||
where: { id: projectId },
|
||||
select: {
|
||||
defaultWorkerGroup: {
|
||||
select: { workloadType: true, masterQueue: true },
|
||||
},
|
||||
organization: {
|
||||
select: { id: true, featureFlags: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!project) {
|
||||
return { mode: "skip", migrated: false, reason: "no-project" };
|
||||
}
|
||||
|
||||
if (project.defaultWorkerGroup?.workloadType === "MICROVM") {
|
||||
return { mode: "required", migrated: false, reason: "microvm-native" };
|
||||
}
|
||||
|
||||
// Migrated orgs route runs to the compute backing even though their stored
|
||||
// default is still the container region, so they need a compute template too.
|
||||
// shadow mode: never fail a deploy over a backing the org didn't opt into.
|
||||
// A cold registry read returns no backing, so this is simply skipped until loaded.
|
||||
const defaultQueue = project.defaultWorkerGroup?.masterQueue;
|
||||
if (defaultQueue && backingForQueue(defaultQueue, workerRegionRegistry.current() ?? [])) {
|
||||
const decision = {
|
||||
orgId: project.organization.id,
|
||||
orgFeatureFlags: project.organization.featureFlags as Record<string, unknown> | null,
|
||||
flags: globalFlagsRegistry.current(),
|
||||
};
|
||||
// Per-org override needs no plan; only the percentage path does. So skip the
|
||||
// external entitlement lookup unless it could matter, and degrade gracefully
|
||||
// if it throws - a shadow-template check must never fail a deploy.
|
||||
let migrated = isOrgMigrated({ ...decision, planType: undefined });
|
||||
if (!migrated && (decision.flags?.computeMigrationEnabled ?? false)) {
|
||||
let planType: string | undefined;
|
||||
try {
|
||||
planType = (await getEntitlement(project.organization.id))?.plan?.type;
|
||||
} catch (error) {
|
||||
logger.warn("compute migration: entitlement lookup failed; skipping shadow template", {
|
||||
organizationId: project.organization.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
migrated = isOrgMigrated({ ...decision, planType });
|
||||
}
|
||||
if (migrated) {
|
||||
// required => template built at deploy (deploy fails on error); off => shadow.
|
||||
return {
|
||||
mode: decision.flags?.computeMigrationRequireTemplate ? "required" : "shadow",
|
||||
migrated: true,
|
||||
reason: "migrated",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const hasComputeAccess = await resolveComputeAccess(prisma, project.organization.featureFlags);
|
||||
|
||||
if (hasComputeAccess) {
|
||||
return { mode: "shadow", migrated: false, reason: "compute-access" };
|
||||
}
|
||||
|
||||
const rolloutPct = Number(env.COMPUTE_TEMPLATE_SHADOW_ROLLOUT_PCT ?? "0");
|
||||
if (rolloutPct > 0 && Math.random() * 100 < rolloutPct) {
|
||||
return { mode: "shadow", migrated: false, reason: "rollout" };
|
||||
}
|
||||
|
||||
return { mode: "skip", migrated: false, reason: "none" };
|
||||
}
|
||||
|
||||
async createTemplate(
|
||||
imageReference: string,
|
||||
options?: { background?: boolean }
|
||||
): Promise<CreateTemplateOutcome> {
|
||||
if (!this.client) {
|
||||
return { error: "Compute gateway not configured", results: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const machineConfigs = this.presets.map((p) => ({
|
||||
cpu: p.cpu,
|
||||
memory_gb: p.memory_gb,
|
||||
}));
|
||||
|
||||
const response = await this.client.templates.create({
|
||||
image: stripImageDigest(imageReference),
|
||||
machine_configs: machineConfigs,
|
||||
background: options?.background,
|
||||
});
|
||||
|
||||
// Background mode (202 Accepted): no body to inspect.
|
||||
if (options?.background || !response) {
|
||||
return { results: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
error: response.error,
|
||||
results: response.results,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
logger.error("Failed to create compute template", {
|
||||
imageReference,
|
||||
error: message,
|
||||
});
|
||||
return { error: message, results: [] };
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a human-readable failure message if any required preset failed
|
||||
// or the request itself failed. Optional preset failures are logged and
|
||||
// do not contribute to the message. Returns undefined on success.
|
||||
private failureMessageForRequiredMode(
|
||||
outcome: CreateTemplateOutcome,
|
||||
deploymentFriendlyId: string,
|
||||
imageReference: string
|
||||
): string | undefined {
|
||||
if (this.presets.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const failures: string[] = [];
|
||||
|
||||
this.presets.forEach((preset) => {
|
||||
const isRequired = this.requiredPresets.has(preset.name);
|
||||
// Match results to presets by (cpu, memory_gb) content with a small
|
||||
// epsilon to tolerate float round-trip noise (memory_gb passes through
|
||||
// gb -> mb -> gb conversion in the compute layer).
|
||||
const result = outcome.results.find(
|
||||
(r) =>
|
||||
Math.abs(r.machine_config.cpu - preset.cpu) < 1e-9 &&
|
||||
Math.abs(r.machine_config.memory_gb - preset.memory_gb) < 1e-9
|
||||
);
|
||||
|
||||
if (!result) {
|
||||
if (isRequired) {
|
||||
failures.push(`${preset.name}: not built`);
|
||||
} else {
|
||||
logger.warn("Optional compute template preset not built", {
|
||||
id: deploymentFriendlyId,
|
||||
imageReference,
|
||||
preset: preset.name,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
if (isRequired) {
|
||||
failures.push(`${preset.name}: ${result.error}`);
|
||||
} else {
|
||||
logger.warn("Optional compute template preset failed", {
|
||||
id: deploymentFriendlyId,
|
||||
imageReference,
|
||||
preset: preset.name,
|
||||
error: result.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Surface request-level errors only when no per-preset failure attributed.
|
||||
if (outcome.error && failures.length === 0) {
|
||||
failures.push(outcome.error);
|
||||
}
|
||||
|
||||
return failures.length > 0 ? failures.join("; ") : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
type CreateTemplateOutcome = {
|
||||
error?: string;
|
||||
results: TemplateCreateResultEntry[];
|
||||
};
|
||||
@@ -0,0 +1,210 @@
|
||||
import type { TaskQueue, User } from "@trigger.dev/database";
|
||||
import { errAsync, fromPromise, okAsync } from "neverthrow";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server";
|
||||
import { engine } from "../runEngine.server";
|
||||
|
||||
export type ConcurrencySystemOptions = {
|
||||
db: PrismaClientOrTransaction;
|
||||
reader: PrismaClientOrTransaction;
|
||||
};
|
||||
|
||||
export type QueueInput = string | { type: "task" | "custom"; name: string };
|
||||
|
||||
export class ConcurrencySystem {
|
||||
constructor(private readonly options: ConcurrencySystemOptions) {}
|
||||
|
||||
private get db() {
|
||||
return this.options.db;
|
||||
}
|
||||
|
||||
get queues() {
|
||||
return {
|
||||
overrideQueueConcurrencyLimit: (
|
||||
environment: AuthenticatedEnvironment,
|
||||
queue: QueueInput,
|
||||
concurrencyLimit: number,
|
||||
overriddenBy?: User
|
||||
) => {
|
||||
return findQueueFromInput(this.db, environment, queue)
|
||||
.andThen((queue) =>
|
||||
overrideQueueConcurrencyLimit(
|
||||
this.db,
|
||||
environment,
|
||||
queue,
|
||||
concurrencyLimit,
|
||||
overriddenBy
|
||||
)
|
||||
)
|
||||
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
|
||||
.andThen((queue) => getQueueStats(environment, queue));
|
||||
},
|
||||
resetConcurrencyLimit: (environment: AuthenticatedEnvironment, queue: QueueInput) => {
|
||||
return findQueueFromInput(this.db, environment, queue)
|
||||
.andThen((queue) => resetQueueConcurrencyLimit(this.db, queue))
|
||||
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
|
||||
.andThen((queue) => getQueueStats(environment, queue));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function findQueueFromInput(
|
||||
db: PrismaClientOrTransaction,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queue: QueueInput
|
||||
) {
|
||||
if (typeof queue === "string") {
|
||||
return findQueueByFriendlyId(db, environment, queue);
|
||||
}
|
||||
|
||||
const queueName =
|
||||
queue.type === "task" ? `task/${queue.name.replace(/^task\//, "")}` : queue.name;
|
||||
|
||||
return findQueueByName(db, environment, queueName);
|
||||
}
|
||||
|
||||
function findQueueByFriendlyId(
|
||||
db: PrismaClientOrTransaction,
|
||||
environment: AuthenticatedEnvironment,
|
||||
friendlyId: string
|
||||
) {
|
||||
return fromPromise(
|
||||
db.taskQueue.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
friendlyId,
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen((queue) => {
|
||||
if (!queue) {
|
||||
return errAsync({ type: "queue_not_found" as const });
|
||||
}
|
||||
return okAsync(queue);
|
||||
});
|
||||
}
|
||||
|
||||
function findQueueByName(
|
||||
db: PrismaClientOrTransaction,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queue: string
|
||||
) {
|
||||
return fromPromise(
|
||||
db.taskQueue.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
name: queue,
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen((queue) => {
|
||||
if (!queue) {
|
||||
return errAsync({ type: "queue_not_found" as const });
|
||||
}
|
||||
return okAsync(queue);
|
||||
});
|
||||
}
|
||||
|
||||
function overrideQueueConcurrencyLimit(
|
||||
db: PrismaClientOrTransaction,
|
||||
environment: AuthenticatedEnvironment,
|
||||
queue: TaskQueue,
|
||||
concurrencyLimit: number,
|
||||
overriddenBy?: User
|
||||
) {
|
||||
const newConcurrencyLimit = Math.max(
|
||||
Math.min(concurrencyLimit, environment.maximumConcurrencyLimit),
|
||||
0
|
||||
);
|
||||
|
||||
const concurrencyLimitBase = queue.concurrencyLimitOverriddenAt
|
||||
? queue.concurrencyLimitBase
|
||||
: queue.concurrencyLimit;
|
||||
|
||||
return fromPromise(
|
||||
db.taskQueue.update({
|
||||
where: {
|
||||
id: queue.id,
|
||||
},
|
||||
data: {
|
||||
concurrencyLimit: newConcurrencyLimit,
|
||||
concurrencyLimitBase: concurrencyLimitBase ?? null,
|
||||
concurrencyLimitOverriddenAt: new Date(),
|
||||
concurrencyLimitOverriddenBy: overriddenBy?.id ?? null,
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "queue_update_failed" as const,
|
||||
cause: error,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function resetQueueConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQueue) {
|
||||
if (queue.concurrencyLimitOverriddenAt === null) {
|
||||
return errAsync({ type: "queue_not_overridden" as const });
|
||||
}
|
||||
|
||||
const newConcurrencyLimit = queue.concurrencyLimitBase;
|
||||
|
||||
return fromPromise(
|
||||
db.taskQueue.update({
|
||||
where: { id: queue.id },
|
||||
data: {
|
||||
concurrencyLimitOverriddenAt: null,
|
||||
concurrencyLimit: newConcurrencyLimit,
|
||||
concurrencyLimitBase: null,
|
||||
concurrencyLimitOverriddenBy: null,
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "queue_update_failed" as const,
|
||||
cause: error,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function syncQueueConcurrencyToEngine(environment: AuthenticatedEnvironment, queue: TaskQueue) {
|
||||
if (queue.paused) {
|
||||
// Queue is paused, don't update Redis limits - keep at 0
|
||||
return okAsync(queue);
|
||||
}
|
||||
|
||||
if (typeof queue.concurrencyLimit === "number") {
|
||||
return fromPromise(
|
||||
updateQueueConcurrencyLimits(environment, queue.name, queue.concurrencyLimit),
|
||||
(error) => ({
|
||||
type: "sync_queue_concurrency_to_engine_failed" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen(() => okAsync(queue));
|
||||
} else {
|
||||
return fromPromise(removeQueueConcurrencyLimits(environment, queue.name), (error) => ({
|
||||
type: "sync_queue_concurrency_to_engine_failed" as const,
|
||||
cause: error,
|
||||
})).andThen(() => okAsync(queue));
|
||||
}
|
||||
}
|
||||
|
||||
function getQueueStats(environment: AuthenticatedEnvironment, queue: TaskQueue) {
|
||||
return fromPromise(
|
||||
Promise.all([
|
||||
engine.lengthOfQueues(environment, [queue.name]),
|
||||
engine.currentConcurrencyOfQueues(environment, [queue.name]),
|
||||
]),
|
||||
(error) => ({
|
||||
type: "get_queue_stats_failed" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen(([queued, running]) =>
|
||||
okAsync({ queued: queued[queue.name] ?? 0, running: running[queue.name] ?? 0, ...queue })
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { prisma, $replica } from "~/db.server";
|
||||
import { ConcurrencySystem } from "./concurrencySystem.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
export const concurrencySystem = singleton(
|
||||
"concurrency-system",
|
||||
initalizeConcurrencySystemInstance
|
||||
);
|
||||
|
||||
function initalizeConcurrencySystemInstance() {
|
||||
return new ConcurrencySystem({
|
||||
db: prisma,
|
||||
reader: $replica,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import type { TaskRunInternalError } from "@trigger.dev/core/v3";
|
||||
import { sanitizeError, TaskRunErrorCodes } from "@trigger.dev/core/v3";
|
||||
import type { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
|
||||
import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
import { getEventRepositoryForStore } from "../eventRepository/index.server";
|
||||
|
||||
export type CrashTaskRunServiceOptions = {
|
||||
reason?: string;
|
||||
exitCode?: number;
|
||||
logs?: string;
|
||||
crashAttempts?: boolean;
|
||||
crashedAt?: Date;
|
||||
overrideCompletion?: boolean;
|
||||
errorCode?: TaskRunInternalError["code"];
|
||||
};
|
||||
|
||||
export class CrashTaskRunService extends BaseService {
|
||||
public async call(runId: string, options?: CrashTaskRunServiceOptions) {
|
||||
const opts = {
|
||||
reason: "Worker crashed",
|
||||
crashAttempts: true,
|
||||
crashedAt: new Date(),
|
||||
...options,
|
||||
};
|
||||
|
||||
logger.debug("CrashTaskRunService.call", { runId, opts });
|
||||
|
||||
if (options?.overrideCompletion) {
|
||||
logger.error("CrashTaskRunService.call: overrideCompletion is deprecated", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
const taskRun = await this.runStore.findRun({ id: runId }, this._prisma);
|
||||
|
||||
if (!taskRun) {
|
||||
logger.error("[CrashTaskRunService] Task run not found", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
// Make sure the task run is in a crashable state
|
||||
if (!opts.overrideCompletion && !isCrashableRunStatus(taskRun.status)) {
|
||||
logger.error("[CrashTaskRunService] Task run is not in a crashable state", {
|
||||
runId,
|
||||
status: taskRun.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("[CrashTaskRunService] Completing attempt", { runId, options });
|
||||
|
||||
const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
|
||||
const retryResult = await retryHelper.call({
|
||||
runId,
|
||||
completion: {
|
||||
ok: false,
|
||||
id: runId,
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
|
||||
message: opts.reason,
|
||||
stackTrace: opts.logs,
|
||||
},
|
||||
},
|
||||
isCrash: true,
|
||||
});
|
||||
|
||||
logger.debug("[CrashTaskRunService] Completion result", { runId, retryResult });
|
||||
|
||||
if (retryResult === "RETRIED") {
|
||||
logger.debug("[CrashTaskRunService] Retried task run", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!opts.overrideCompletion) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("[CrashTaskRunService] Overriding completion", { runId, options });
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
const crashedTaskRun = await finalizeService.call({
|
||||
id: taskRun.id,
|
||||
status: "CRASHED",
|
||||
completedAt: new Date(),
|
||||
include: {
|
||||
attempts: {
|
||||
where: {
|
||||
status: {
|
||||
in: CRASHABLE_ATTEMPT_STATUSES,
|
||||
},
|
||||
},
|
||||
include: {
|
||||
backgroundWorker: true,
|
||||
runtimeEnvironment: true,
|
||||
},
|
||||
},
|
||||
dependency: true,
|
||||
runtimeEnvironment: {
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
attemptStatus: "FAILED",
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
|
||||
message: opts.reason,
|
||||
stackTrace: opts.logs,
|
||||
},
|
||||
});
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
crashedTaskRun.taskEventStore,
|
||||
crashedTaskRun.runtimeEnvironment.organizationId
|
||||
);
|
||||
|
||||
const [createAttemptFailedEventError] = await tryCatch(
|
||||
eventRepository.completeFailedRunEvent({
|
||||
run: crashedTaskRun,
|
||||
endTime: opts.crashedAt,
|
||||
exception: {
|
||||
type: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
|
||||
message: opts.reason,
|
||||
stacktrace: opts.logs,
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
if (createAttemptFailedEventError) {
|
||||
logger.error("[CrashTaskRunService] Failed to complete failed run event", {
|
||||
error: createAttemptFailedEventError,
|
||||
runId: crashedTaskRun.id,
|
||||
});
|
||||
}
|
||||
|
||||
if (!opts.crashAttempts) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Cancel any in progress attempts
|
||||
for (const attempt of crashedTaskRun.attempts) {
|
||||
await this.#failAttempt(
|
||||
attempt,
|
||||
crashedTaskRun,
|
||||
new Date(),
|
||||
crashedTaskRun.runtimeEnvironment,
|
||||
{
|
||||
reason: opts.reason,
|
||||
logs: opts.logs,
|
||||
code: opts.errorCode,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #failAttempt(
|
||||
attempt: TaskRunAttempt,
|
||||
run: TaskRun,
|
||||
failedAt: Date,
|
||||
environment: AuthenticatedEnvironment,
|
||||
error: {
|
||||
reason: string;
|
||||
logs?: string;
|
||||
code?: TaskRunInternalError["code"];
|
||||
}
|
||||
) {
|
||||
return await this.traceWithEnv(
|
||||
"[CrashTaskRunService] failAttempt()",
|
||||
environment,
|
||||
async (span) => {
|
||||
span.setAttribute("taskRunId", run.id);
|
||||
span.setAttribute("attemptId", attempt.id);
|
||||
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: {
|
||||
id: attempt.id,
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
completedAt: failedAt,
|
||||
error: sanitizeError({
|
||||
type: "INTERNAL_ERROR",
|
||||
code: error.code ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
|
||||
message: error.reason,
|
||||
stackTrace: error.logs,
|
||||
}),
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,990 @@
|
||||
import type {
|
||||
BackgroundWorkerMetadata,
|
||||
BackgroundWorkerSourceFileMetadata,
|
||||
CreateBackgroundWorkerRequestBody,
|
||||
PromptResource,
|
||||
QueueManifest,
|
||||
TaskResource,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { tryCatch } from "@trigger.dev/core/v3";
|
||||
import { BackgroundWorkerId, stringifyDuration } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { BackgroundWorker, TaskQueue, TaskQueueType } from "@trigger.dev/database";
|
||||
import cronstrue from "cronstrue";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { $transaction, Prisma } from "~/db.server";
|
||||
import { sanitizeQueueName } from "~/models/taskQueue.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { syncTaskIdentifiers } from "~/services/taskIdentifierRegistry.server";
|
||||
import {
|
||||
type TaskMetadataCache,
|
||||
type TaskMetadataEntry,
|
||||
} from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { engine } from "../runEngine.server";
|
||||
import {
|
||||
removeQueueConcurrencyLimits,
|
||||
updateEnvConcurrencyLimits,
|
||||
updateQueueConcurrencyLimits,
|
||||
} from "../runQueue.server";
|
||||
import { scheduleEngine } from "../scheduleEngine.server";
|
||||
import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion";
|
||||
import { clampMaxDuration } from "../utils/maxDuration";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { CheckScheduleService } from "./checkSchedule.server";
|
||||
import { projectPubSub } from "./projectPubSub.server";
|
||||
|
||||
import { assertNoDuplicateTaskIds } from "./duplicateTaskIds.server";
|
||||
import { stripBackgroundWorkerMetadataForStorage } from "./stripBackgroundWorkerMetadataForStorage.server";
|
||||
export { stripBackgroundWorkerMetadataForStorage };
|
||||
|
||||
export class CreateBackgroundWorkerService extends BaseService {
|
||||
private readonly _taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
prisma?: PrismaClientOrTransaction,
|
||||
replica?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
super(prisma, replica);
|
||||
this._taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
public async call(
|
||||
projectRef: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: CreateBackgroundWorkerRequestBody
|
||||
): Promise<BackgroundWorker> {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
span.setAttribute("projectRef", projectRef);
|
||||
|
||||
const project = await this._prisma.project.findFirstOrThrow({
|
||||
where: {
|
||||
externalRef: projectRef,
|
||||
environments: {
|
||||
some: {
|
||||
id: environment.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
include: {
|
||||
backgroundWorkers: {
|
||||
where: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
take: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const latestBackgroundWorker = project.backgroundWorkers[0];
|
||||
|
||||
if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) {
|
||||
return latestBackgroundWorker;
|
||||
}
|
||||
|
||||
const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version);
|
||||
|
||||
logger.debug(`Creating background worker`, {
|
||||
nextVersion,
|
||||
lastVersion: project.backgroundWorkers[0]?.version,
|
||||
});
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.create({
|
||||
data: {
|
||||
...BackgroundWorkerId.generate(),
|
||||
version: nextVersion,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
metadata: stripBackgroundWorkerMetadataForStorage(body.metadata),
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
runtime: body.metadata.runtime,
|
||||
runtimeVersion: body.metadata.runtimeVersion,
|
||||
supportsLazyAttempts: body.supportsLazyAttempts,
|
||||
engine: body.engine,
|
||||
},
|
||||
});
|
||||
|
||||
//upgrade the project to engine "V2" if it's not already
|
||||
if (project.engine === "V1" && body.engine === "V2") {
|
||||
await this._prisma.project.update({
|
||||
where: {
|
||||
id: project.id,
|
||||
},
|
||||
data: {
|
||||
engine: "V2",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const [filesError, tasksToBackgroundFiles] = await tryCatch(
|
||||
createBackgroundFiles(
|
||||
body.metadata.sourceFiles,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma
|
||||
)
|
||||
);
|
||||
|
||||
if (filesError) {
|
||||
logger.error("Error creating background worker files", {
|
||||
error: filesError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
|
||||
throw new ServiceValidationError("Error creating background worker files");
|
||||
}
|
||||
|
||||
const [resourcesError, workerTaskEntries] = await tryCatch(
|
||||
createWorkerResources(
|
||||
body.metadata,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma,
|
||||
tasksToBackgroundFiles
|
||||
)
|
||||
);
|
||||
|
||||
if (resourcesError) {
|
||||
if (resourcesError instanceof ServiceValidationError) {
|
||||
// Customer-facing config error (e.g. duplicate task ids). Surface the
|
||||
// real message to the client via the rethrow.
|
||||
logger.warn("Error creating worker resources", {
|
||||
error: resourcesError.message,
|
||||
});
|
||||
throw resourcesError;
|
||||
}
|
||||
|
||||
logger.error("Error creating worker resources", {
|
||||
error: resourcesError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
throw new ServiceValidationError("Error creating worker resources");
|
||||
}
|
||||
|
||||
const [schedulesError] = await tryCatch(
|
||||
syncDeclarativeSchedules(body.metadata.tasks, backgroundWorker, environment, this._prisma)
|
||||
);
|
||||
|
||||
if (schedulesError) {
|
||||
if (schedulesError instanceof ServiceValidationError) {
|
||||
// Customer schedule config (typically invalid cron). Surface to
|
||||
// client via the rethrow; system returns gracefully.
|
||||
logger.warn("Error syncing declarative schedules", {
|
||||
error: schedulesError.message,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
throw schedulesError;
|
||||
}
|
||||
|
||||
// Wrapping the underlying error into a ServiceValidationError below
|
||||
// would otherwise hide it once the SDK-level filter drops SVEs; log at
|
||||
// error so the underlying cause stays visible. Mirrors the
|
||||
// waitpointCompletionPacket.server.ts pattern from dac9c83bd.
|
||||
logger.error("Error syncing declarative schedules", {
|
||||
error: schedulesError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
|
||||
throw new ServiceValidationError("Error syncing declarative schedules");
|
||||
}
|
||||
|
||||
const [syncIdentifiersError] = await tryCatch(
|
||||
syncTaskIdentifiers(
|
||||
environment.id,
|
||||
project.id,
|
||||
backgroundWorker.id,
|
||||
body.metadata.tasks.map((t) => ({ id: t.id, triggerSource: t.triggerSource }))
|
||||
)
|
||||
);
|
||||
|
||||
if (syncIdentifiersError) {
|
||||
logger.error("Error syncing task identifiers", {
|
||||
error: syncIdentifiersError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
}
|
||||
|
||||
// Populate task metadata cache. DEV workers are always "current" because
|
||||
// `findCurrentWorkerFromEnvironment` resolves DEV current as the latest
|
||||
// worker by createdAt. Non-DEV (deploy-built) workers are not promoted
|
||||
// here — promotion writes the `:env:` keyspace later in
|
||||
// changeCurrentDeployment / createDeploymentBackgroundWorkerV3.
|
||||
// Cache calls log+swallow internally, so a Redis blip can't break
|
||||
// anything else here. Empty `workerTaskEntries` is intentional — the
|
||||
// populate methods clear stale hashes for zero-task deploys.
|
||||
if (workerTaskEntries) {
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
await this._taskMetaCache.populateByCurrentWorker(
|
||||
environment.id,
|
||||
backgroundWorker.id,
|
||||
workerTaskEntries
|
||||
);
|
||||
} else {
|
||||
await this._taskMetaCache.populateByWorker(backgroundWorker.id, workerTaskEntries);
|
||||
}
|
||||
}
|
||||
|
||||
const [updateConcurrencyLimitsError] = await tryCatch(
|
||||
updateEnvConcurrencyLimits(environment)
|
||||
);
|
||||
|
||||
if (updateConcurrencyLimitsError) {
|
||||
logger.error("Error updating environment concurrency limits", {
|
||||
error: updateConcurrencyLimitsError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
}
|
||||
|
||||
const [publishError] = await tryCatch(
|
||||
projectPubSub.publish(`project:${project.id}:env:${environment.id}`, "WORKER_CREATED", {
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
createdAt: backgroundWorker.createdAt,
|
||||
taskCount: body.metadata.tasks.length,
|
||||
type: "local",
|
||||
})
|
||||
);
|
||||
|
||||
if (publishError) {
|
||||
logger.error("Error publishing WORKER_CREATED event", {
|
||||
error: publishError,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
});
|
||||
}
|
||||
|
||||
if (backgroundWorker.engine === "V2") {
|
||||
const [schedulePendingVersionsError] = await tryCatch(
|
||||
engine.scheduleEnqueueRunsForBackgroundWorker(backgroundWorker.id)
|
||||
);
|
||||
|
||||
if (schedulePendingVersionsError) {
|
||||
logger.error("Error scheduling pending versions", {
|
||||
error: schedulePendingVersionsError,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return backgroundWorker;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function createWorkerResources(
|
||||
metadata: BackgroundWorkerMetadata,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction,
|
||||
tasksToBackgroundFiles?: Map<string, string>
|
||||
): Promise<TaskMetadataEntry[]> {
|
||||
// Defense-in-depth against two tasks sharing an id (across all task types,
|
||||
// e.g. a schedule and a regular task). Note: the CLI's resource catalog keys
|
||||
// tasks by id and overwrites collisions, so duplicates are normally already
|
||||
// collapsed before reaching here — this guards against any client that sends
|
||||
// an un-deduplicated task list.
|
||||
assertNoDuplicateTaskIds(metadata.tasks);
|
||||
|
||||
// Create the queues
|
||||
const queues = await createWorkerQueues(metadata, worker, environment, prisma);
|
||||
|
||||
// Create the tasks
|
||||
const taskEntries = await createWorkerTasks(
|
||||
metadata,
|
||||
queues,
|
||||
worker,
|
||||
environment,
|
||||
prisma,
|
||||
tasksToBackgroundFiles
|
||||
);
|
||||
|
||||
// Register prompts
|
||||
if (metadata.prompts && metadata.prompts.length > 0) {
|
||||
await createWorkerPrompts(metadata.prompts, worker, environment, prisma);
|
||||
}
|
||||
|
||||
return taskEntries;
|
||||
}
|
||||
|
||||
async function createWorkerTasks(
|
||||
metadata: BackgroundWorkerMetadata,
|
||||
queues: Array<TaskQueue>,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction,
|
||||
tasksToBackgroundFiles?: Map<string, string>
|
||||
): Promise<TaskMetadataEntry[]> {
|
||||
// Create tasks in chunks of 20
|
||||
const CHUNK_SIZE = 20;
|
||||
const entries: TaskMetadataEntry[] = [];
|
||||
for (let i = 0; i < metadata.tasks.length; i += CHUNK_SIZE) {
|
||||
const chunk = metadata.tasks.slice(i, i + CHUNK_SIZE);
|
||||
const chunkEntries = await Promise.all(
|
||||
chunk.map((task) =>
|
||||
createWorkerTask(task, queues, worker, environment, prisma, tasksToBackgroundFiles)
|
||||
)
|
||||
);
|
||||
for (const entry of chunkEntries) {
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function createWorkerTask(
|
||||
task: TaskResource,
|
||||
queues: Array<TaskQueue>,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction,
|
||||
tasksToBackgroundFiles?: Map<string, string>
|
||||
): Promise<TaskMetadataEntry | null> {
|
||||
// Hoisted so the P2002 catch branch can return the same entry shape.
|
||||
let queue: TaskQueue | undefined;
|
||||
let resolvedTriggerSource: "SCHEDULED" | "AGENT" | "STANDARD" | undefined;
|
||||
let resolvedTtl: string | null | undefined;
|
||||
|
||||
try {
|
||||
queue = queues.find((queue) => queue.name === task.queue?.name);
|
||||
|
||||
if (!queue) {
|
||||
// Create a TaskQueue
|
||||
queue = await createWorkerQueue(
|
||||
{
|
||||
name: task.queue?.name ?? `task/${task.id}`,
|
||||
concurrencyLimit: task.queue?.concurrencyLimit,
|
||||
},
|
||||
task.id,
|
||||
task.queue?.name ? "NAMED" : "VIRTUAL",
|
||||
worker,
|
||||
environment,
|
||||
prisma
|
||||
);
|
||||
}
|
||||
|
||||
resolvedTriggerSource =
|
||||
task.triggerSource === "schedule"
|
||||
? ("SCHEDULED" as const)
|
||||
: task.triggerSource === "agent"
|
||||
? ("AGENT" as const)
|
||||
: ("STANDARD" as const);
|
||||
|
||||
resolvedTtl =
|
||||
typeof task.ttl === "number" ? (stringifyDuration(task.ttl) ?? null) : (task.ttl ?? null);
|
||||
|
||||
await prisma.backgroundWorkerTask.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("task"),
|
||||
projectId: worker.projectId,
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
workerId: worker.id,
|
||||
slug: task.id,
|
||||
description: task.description,
|
||||
filePath: task.filePath,
|
||||
exportName: task.exportName,
|
||||
retryConfig: task.retry,
|
||||
queueConfig: task.queue,
|
||||
machineConfig: task.machine,
|
||||
triggerSource: resolvedTriggerSource,
|
||||
config: task.agentConfig ? (task.agentConfig as any) : undefined,
|
||||
fileId: tasksToBackgroundFiles?.get(task.id) ?? null,
|
||||
maxDurationInSeconds: task.maxDuration ? clampMaxDuration(task.maxDuration) : null,
|
||||
ttl: resolvedTtl,
|
||||
queueId: queue.id,
|
||||
payloadSchema: task.payloadSchema as any,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
slug: task.id,
|
||||
ttl: resolvedTtl,
|
||||
triggerSource: resolvedTriggerSource,
|
||||
queueId: queue.id,
|
||||
queueName: queue.name,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
// The error code for unique constraint violation in Prisma is P2002
|
||||
if (error.code === "P2002") {
|
||||
// Retry landing after the first attempt's row was already written.
|
||||
const existing = await prisma.backgroundWorkerTask.findFirst({
|
||||
where: { workerId: worker.id, slug: task.id },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
logger.warn("Attempted to recreate background worker task", {
|
||||
task,
|
||||
worker,
|
||||
});
|
||||
|
||||
if (existing && queue && resolvedTriggerSource && resolvedTtl !== undefined) {
|
||||
return {
|
||||
slug: task.id,
|
||||
ttl: resolvedTtl,
|
||||
triggerSource: resolvedTriggerSource,
|
||||
queueId: queue.id,
|
||||
queueName: queue.name,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
logger.error("Prisma Error creating background worker task", {
|
||||
error: {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
},
|
||||
task,
|
||||
worker,
|
||||
});
|
||||
}
|
||||
} else if (error instanceof Error) {
|
||||
logger.error("Error creating background worker task", {
|
||||
error: {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
},
|
||||
task,
|
||||
worker,
|
||||
});
|
||||
} else {
|
||||
logger.error("Unknown error creating background worker task", {
|
||||
error,
|
||||
task,
|
||||
worker,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function createWorkerQueues(
|
||||
metadata: BackgroundWorkerMetadata,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
if (!metadata.queues) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const CHUNK_SIZE = 20;
|
||||
const allQueues: Awaited<ReturnType<typeof createWorkerQueue>>[] = [];
|
||||
|
||||
// Process queues in chunks
|
||||
for (let i = 0; i < metadata.queues.length; i += CHUNK_SIZE) {
|
||||
const chunk = metadata.queues.slice(i, i + CHUNK_SIZE);
|
||||
const queueChunk = await Promise.all(
|
||||
chunk.map(async (queue) => {
|
||||
return createWorkerQueue(queue, queue.name, "NAMED", worker, environment, prisma);
|
||||
})
|
||||
);
|
||||
allQueues.push(...queueChunk.filter(Boolean));
|
||||
}
|
||||
|
||||
return allQueues;
|
||||
}
|
||||
|
||||
async function createWorkerQueue(
|
||||
queue: QueueManifest,
|
||||
orderableName: string,
|
||||
queueType: TaskQueueType,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
let queueName = sanitizeQueueName(queue.name);
|
||||
|
||||
const baseConcurrencyLimit =
|
||||
typeof queue.concurrencyLimit === "number"
|
||||
? Math.max(Math.min(queue.concurrencyLimit, environment.maximumConcurrencyLimit), 0)
|
||||
: queue.concurrencyLimit;
|
||||
|
||||
const taskQueue = await upsertWorkerQueueRecord(
|
||||
queueName,
|
||||
baseConcurrencyLimit ?? null,
|
||||
orderableName,
|
||||
queueType,
|
||||
worker,
|
||||
prisma
|
||||
);
|
||||
|
||||
const newConcurrencyLimit = taskQueue.concurrencyLimit;
|
||||
|
||||
if (!taskQueue.paused) {
|
||||
if (typeof newConcurrencyLimit === "number") {
|
||||
logger.debug("createWorkerQueue: updating concurrency limit", {
|
||||
workerId: worker.id,
|
||||
taskQueue,
|
||||
orgId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
concurrencyLimit: newConcurrencyLimit,
|
||||
});
|
||||
await updateQueueConcurrencyLimits(environment, taskQueue.name, newConcurrencyLimit);
|
||||
} else {
|
||||
logger.debug("createWorkerQueue: removing concurrency limit", {
|
||||
workerId: worker.id,
|
||||
taskQueue,
|
||||
orgId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
concurrencyLimit: newConcurrencyLimit,
|
||||
});
|
||||
await removeQueueConcurrencyLimits(environment, taskQueue.name);
|
||||
}
|
||||
} else {
|
||||
logger.debug("createWorkerQueue: queue is paused, not updating concurrency limit", {
|
||||
workerId: worker.id,
|
||||
taskQueue,
|
||||
orgId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
}
|
||||
|
||||
return taskQueue;
|
||||
}
|
||||
|
||||
async function upsertWorkerQueueRecord(
|
||||
queueName: string,
|
||||
concurrencyLimit: number | null,
|
||||
orderableName: string,
|
||||
queueType: TaskQueueType,
|
||||
worker: BackgroundWorker,
|
||||
prisma: PrismaClientOrTransaction,
|
||||
attempt: number = 0
|
||||
): Promise<TaskQueue> {
|
||||
if (attempt > 3) {
|
||||
throw new Error("Failed to insert queue record");
|
||||
}
|
||||
|
||||
try {
|
||||
let taskQueue = await prisma.taskQueue.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
name: queueName,
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskQueue) {
|
||||
taskQueue = await prisma.taskQueue.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("queue"),
|
||||
version: "V2",
|
||||
name: queueName,
|
||||
orderableName,
|
||||
concurrencyLimit,
|
||||
runtimeEnvironmentId: worker.runtimeEnvironmentId,
|
||||
projectId: worker.projectId,
|
||||
type: queueType,
|
||||
workers: {
|
||||
connect: {
|
||||
id: worker.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} else {
|
||||
const hasOverride = taskQueue.concurrencyLimitOverriddenAt !== null;
|
||||
|
||||
taskQueue = await prisma.taskQueue.update({
|
||||
where: {
|
||||
id: taskQueue.id,
|
||||
},
|
||||
data: {
|
||||
workers: { connect: { id: worker.id } },
|
||||
version: "V2",
|
||||
orderableName,
|
||||
// If overridden, keep current limit and update base; otherwise update limit normally
|
||||
concurrencyLimit: hasOverride ? undefined : concurrencyLimit,
|
||||
concurrencyLimitBase: hasOverride ? concurrencyLimit : undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return taskQueue;
|
||||
} catch (error) {
|
||||
// If the queue already exists, let's try again
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
|
||||
return await upsertWorkerQueueRecord(
|
||||
queueName,
|
||||
concurrencyLimit,
|
||||
orderableName,
|
||||
queueType,
|
||||
worker,
|
||||
prisma,
|
||||
attempt + 1
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
//CreateDeclarativeScheduleError with a message
|
||||
export class CreateDeclarativeScheduleError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "CreateDeclarativeScheduleError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncDeclarativeSchedules(
|
||||
tasks: TaskResource[],
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
const tasksWithDeclarativeSchedules = tasks.filter((task) => task.schedule);
|
||||
logger.info("Syncing declarative schedules", {
|
||||
tasksWithDeclarativeSchedules,
|
||||
environment,
|
||||
});
|
||||
|
||||
const existingDeclarativeSchedules = await prisma.taskSchedule.findMany({
|
||||
where: {
|
||||
type: "DECLARATIVE",
|
||||
projectId: environment.projectId,
|
||||
},
|
||||
include: {
|
||||
instances: true,
|
||||
},
|
||||
});
|
||||
|
||||
const checkSchedule = new CheckScheduleService(prisma);
|
||||
|
||||
//start out by assuming they're all missing
|
||||
const missingSchedules = new Set<string>(
|
||||
existingDeclarativeSchedules.map((schedule) => schedule.id)
|
||||
);
|
||||
|
||||
//create/update schedules (+ instances)
|
||||
for (const task of tasksWithDeclarativeSchedules) {
|
||||
if (task.schedule === undefined) continue;
|
||||
|
||||
// Check if this schedule should be created in the current environment
|
||||
if (task.schedule.environments && task.schedule.environments.length > 0) {
|
||||
if (!task.schedule.environments.includes(environment.type)) {
|
||||
logger.debug("Skipping schedule creation due to environment filter", {
|
||||
taskId: task.id,
|
||||
environmentType: environment.type,
|
||||
allowedEnvironments: task.schedule.environments,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const existingSchedule = existingDeclarativeSchedules.find(
|
||||
(schedule) =>
|
||||
schedule.taskIdentifier === task.id &&
|
||||
schedule.instances.some((instance) => instance.environmentId === environment.id)
|
||||
);
|
||||
|
||||
//this throws errors if the schedule is invalid
|
||||
await checkSchedule.call(
|
||||
environment.projectId,
|
||||
{
|
||||
cron: task.schedule.cron,
|
||||
timezone: task.schedule.timezone,
|
||||
taskIdentifier: task.id,
|
||||
friendlyId: existingSchedule?.friendlyId,
|
||||
},
|
||||
[environment.id]
|
||||
);
|
||||
|
||||
if (existingSchedule) {
|
||||
const schedule = await prisma.taskSchedule.update({
|
||||
where: {
|
||||
id: existingSchedule.id,
|
||||
},
|
||||
data: {
|
||||
generatorExpression: task.schedule.cron,
|
||||
generatorDescription: cronstrue.toString(task.schedule.cron),
|
||||
timezone: task.schedule.timezone,
|
||||
},
|
||||
include: {
|
||||
instances: true,
|
||||
},
|
||||
});
|
||||
|
||||
missingSchedules.delete(existingSchedule.id);
|
||||
const instance = schedule.instances.at(0);
|
||||
if (instance) {
|
||||
await scheduleEngine.registerNextTaskScheduleInstance({ instanceId: instance.id });
|
||||
} else {
|
||||
throw new CreateDeclarativeScheduleError(
|
||||
`Missing instance for declarative schedule ${schedule.id}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
const newSchedule = await prisma.taskSchedule.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("sched"),
|
||||
projectId: environment.projectId,
|
||||
taskIdentifier: task.id,
|
||||
generatorExpression: task.schedule.cron,
|
||||
generatorDescription: cronstrue.toString(task.schedule.cron),
|
||||
timezone: task.schedule.timezone,
|
||||
type: "DECLARATIVE",
|
||||
instances: {
|
||||
create: [
|
||||
{
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
include: {
|
||||
instances: true,
|
||||
},
|
||||
});
|
||||
|
||||
const instance = newSchedule.instances.at(0);
|
||||
|
||||
if (instance) {
|
||||
await scheduleEngine.registerNextTaskScheduleInstance({ instanceId: instance.id });
|
||||
} else {
|
||||
throw new CreateDeclarativeScheduleError(
|
||||
`Missing instance for declarative schedule ${newSchedule.id}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Delete instances for this environment
|
||||
//Delete schedules that have no instances left
|
||||
const potentiallyDeletableSchedules = await prisma.taskSchedule.findMany({
|
||||
where: {
|
||||
id: {
|
||||
in: Array.from(missingSchedules),
|
||||
},
|
||||
},
|
||||
include: {
|
||||
instances: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const schedule of potentiallyDeletableSchedules) {
|
||||
const canDeleteSchedule =
|
||||
schedule.instances.length === 0 ||
|
||||
schedule.instances.every((instance) => instance.environmentId === environment.id);
|
||||
|
||||
if (canDeleteSchedule) {
|
||||
//we can delete schedules with no instances other than ones for the current environment
|
||||
await prisma.taskSchedule.delete({
|
||||
where: {
|
||||
id: schedule.id,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
//otherwise we delete the instance (other environments remain untouched)
|
||||
await prisma.taskScheduleInstance.deleteMany({
|
||||
where: {
|
||||
taskScheduleId: schedule.id,
|
||||
environmentId: environment.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createBackgroundFiles(
|
||||
files: Array<BackgroundWorkerSourceFileMetadata> | undefined,
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
// Maps from each taskId to the backgroundWorkerFileId
|
||||
const results = new Map<string, string>();
|
||||
|
||||
if (!files) {
|
||||
return results;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const backgroundWorkerFile = await prisma.backgroundWorkerFile.upsert({
|
||||
where: {
|
||||
projectId_contentHash: {
|
||||
projectId: environment.projectId,
|
||||
contentHash: file.contentHash,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
friendlyId: generateFriendlyId("file"),
|
||||
projectId: environment.projectId,
|
||||
contentHash: file.contentHash,
|
||||
filePath: file.filePath,
|
||||
contents: Buffer.from(file.contents),
|
||||
backgroundWorkers: {
|
||||
connect: {
|
||||
id: worker.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
update: {
|
||||
backgroundWorkers: {
|
||||
connect: {
|
||||
id: worker.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
for (const taskId of file.taskIds) {
|
||||
results.set(taskId, backgroundWorkerFile.id);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
import { createHash } from "crypto";
|
||||
|
||||
function hashContent(content: string): string {
|
||||
return createHash("sha256").update(content).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
async function createWorkerPrompts(
|
||||
prompts: PromptResource[],
|
||||
worker: BackgroundWorker,
|
||||
environment: AuthenticatedEnvironment,
|
||||
prisma: PrismaClientOrTransaction
|
||||
) {
|
||||
for (const promptResource of prompts) {
|
||||
try {
|
||||
// Upsert the Prompt record (identity + schema)
|
||||
const prompt = await prisma.prompt.upsert({
|
||||
where: {
|
||||
projectId_runtimeEnvironmentId_slug: {
|
||||
projectId: worker.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
slug: promptResource.id,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
friendlyId: generateFriendlyId("prompt"),
|
||||
organizationId: environment.organizationId,
|
||||
projectId: worker.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
slug: promptResource.id,
|
||||
description: promptResource.description,
|
||||
filePath: promptResource.filePath,
|
||||
exportName: promptResource.exportName,
|
||||
variableSchema: promptResource.variableSchema as any,
|
||||
defaultModel: promptResource.model,
|
||||
defaultConfig: promptResource.config as any,
|
||||
},
|
||||
update: {
|
||||
description: promptResource.description,
|
||||
filePath: promptResource.filePath,
|
||||
exportName: promptResource.exportName,
|
||||
variableSchema: promptResource.variableSchema as any,
|
||||
defaultModel: promptResource.model,
|
||||
defaultConfig: promptResource.config as any,
|
||||
},
|
||||
});
|
||||
|
||||
// Compute the version-definition hash for dedup. Includes the model and
|
||||
// config, not just the prompt text, so changing a code prompt's model or
|
||||
// config creates a new version — otherwise a model-only change is silently
|
||||
// skipped and the old model keeps serving.
|
||||
const contentString = promptResource.content ?? "";
|
||||
const contentHash = hashContent(
|
||||
JSON.stringify({
|
||||
content: contentString,
|
||||
model: promptResource.model ?? null,
|
||||
config: promptResource.config ?? null,
|
||||
})
|
||||
);
|
||||
|
||||
// Find the latest version overall (for version numbering) and the latest
|
||||
// code-sourced version (for content dedup). We compare against the latest
|
||||
// code version specifically so that dashboard edits don't interfere with
|
||||
// dedup — if the code hasn't changed since the last deploy, we skip even
|
||||
// if a dashboard edit happened in between.
|
||||
const latestVersion = await prisma.promptVersion.findFirst({
|
||||
where: { promptId: prompt.id },
|
||||
orderBy: { version: "desc" },
|
||||
});
|
||||
|
||||
const latestCodeVersion = await prisma.promptVersion.findFirst({
|
||||
where: { promptId: prompt.id, source: "code" },
|
||||
orderBy: { version: "desc" },
|
||||
});
|
||||
|
||||
if (latestCodeVersion?.contentHash === contentHash) {
|
||||
// Code definition (text + model + config) unchanged since last deploy —
|
||||
// skip creating a new version.
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextVersion = (latestVersion?.version ?? 0) + 1;
|
||||
|
||||
// Determine labels for the new version.
|
||||
// Deploys always move "current" to the new code version. If a dashboard
|
||||
// override exists, it sits on top via the "override" label and the API
|
||||
// serves that instead — so "current" movement is safe.
|
||||
const labels = ["latest", "current"];
|
||||
|
||||
// Wrap label removal + version creation in a transaction so labels
|
||||
// aren't stripped if the create fails (e.g. concurrent deploy race).
|
||||
await $transaction(prisma, async (tx) => {
|
||||
// Remove "latest" label from all existing versions
|
||||
if (latestVersion) {
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", 'latest')
|
||||
WHERE "promptId" = ${prompt.id} AND 'latest' = ANY("labels")
|
||||
`;
|
||||
}
|
||||
|
||||
// Remove "current" from any existing version
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", 'current')
|
||||
WHERE "promptId" = ${prompt.id} AND 'current' = ANY("labels")
|
||||
`;
|
||||
|
||||
await tx.promptVersion.create({
|
||||
data: {
|
||||
promptId: prompt.id,
|
||||
version: nextVersion,
|
||||
textContent: contentString,
|
||||
model: promptResource.model,
|
||||
config: promptResource.config as any,
|
||||
source: "code",
|
||||
contentHash,
|
||||
labels,
|
||||
workerId: worker.id,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
logger.debug("Registered prompt version", {
|
||||
promptSlug: promptResource.id,
|
||||
version: nextVersion,
|
||||
labels,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
|
||||
logger.warn("Prompt version already exists", { prompt: promptResource.id });
|
||||
} else {
|
||||
logger.error("Error creating prompt version", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
prompt: promptResource.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
import type { CoordinatorToPlatformMessages, ManualCheckpointMetadata } from "@trigger.dev/core/v3";
|
||||
import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
|
||||
import type { Checkpoint, CheckpointRestoreEvent } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { isFreezableAttemptStatus, isFreezableRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
|
||||
import { ResumeBatchRunService } from "./resumeBatchRun.server";
|
||||
import { ResumeDependentParentsService } from "./resumeDependentParents.server";
|
||||
import { CheckpointId } from "@trigger.dev/core/v3/isomorphic";
|
||||
|
||||
export class CreateCheckpointService extends BaseService {
|
||||
public async call(
|
||||
params: Omit<
|
||||
InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "CHECKPOINT_CREATED">,
|
||||
"version"
|
||||
>
|
||||
): Promise<
|
||||
| {
|
||||
success: true;
|
||||
checkpoint: Checkpoint;
|
||||
event: CheckpointRestoreEvent;
|
||||
keepRunAlive: boolean;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
keepRunAlive?: boolean;
|
||||
}
|
||||
> {
|
||||
logger.debug(`Creating checkpoint`, params);
|
||||
|
||||
const attempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: {
|
||||
friendlyId: params.attemptFriendlyId,
|
||||
},
|
||||
include: {
|
||||
taskRun: true,
|
||||
backgroundWorker: {
|
||||
select: {
|
||||
id: true,
|
||||
deployment: {
|
||||
select: {
|
||||
imageReference: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!attempt) {
|
||||
logger.error("Attempt not found", params);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
!isFreezableAttemptStatus(attempt.status) ||
|
||||
!isFreezableRunStatus(attempt.taskRun.status)
|
||||
) {
|
||||
logger.error("Unfreezable state", {
|
||||
attempt: {
|
||||
id: attempt.id,
|
||||
status: attempt.status,
|
||||
},
|
||||
run: {
|
||||
id: attempt.taskRunId,
|
||||
status: attempt.taskRun.status,
|
||||
},
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
keepRunAlive: true,
|
||||
};
|
||||
}
|
||||
|
||||
const imageRef = attempt.backgroundWorker.deployment?.imageReference;
|
||||
|
||||
if (!imageRef) {
|
||||
logger.error("Missing deployment or image ref", {
|
||||
attemptId: attempt.id,
|
||||
workerId: attempt.backgroundWorker.id,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
};
|
||||
}
|
||||
|
||||
const { reason } = params;
|
||||
|
||||
// Check if we should accept this checkpoint
|
||||
switch (reason.type) {
|
||||
case "MANUAL": {
|
||||
// Always accept manual checkpoints
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_DURATION": {
|
||||
// Always accept duration checkpoints
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_TASK": {
|
||||
const childRun = await this._prisma.taskRun.findFirst({
|
||||
where: {
|
||||
friendlyId: reason.friendlyId,
|
||||
},
|
||||
select: {
|
||||
dependency: {
|
||||
select: {
|
||||
resumedAt: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!childRun) {
|
||||
logger.error("CreateCheckpointService: Pre-check - WAIT_FOR_TASK child run not found", {
|
||||
friendlyId: reason.friendlyId,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (childRun.dependency?.resumedAt) {
|
||||
logger.info("CreateCheckpointService: Child run already resumed", {
|
||||
childRun,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
keepRunAlive: true,
|
||||
};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_BATCH": {
|
||||
// Routed by friendlyId so a run-ops id (NEW-resident) batch is found on the owning DB;
|
||||
// env-scoped to the dependent attempt's run (a batch shares its dependent's env). Read the
|
||||
// primary: a batch that just resumed the parent may lag the replica, and a stale resumedAt
|
||||
// (null) would checkpoint (suspend) an already-resumed run -> it stalls until a sweep.
|
||||
const batchRun = await this.runStore.findBatchTaskRunByFriendlyId(
|
||||
reason.batchFriendlyId,
|
||||
attempt.taskRun.runtimeEnvironmentId,
|
||||
undefined,
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!batchRun) {
|
||||
logger.error("CreateCheckpointService: Pre-check - Batch not found", {
|
||||
batchFriendlyId: reason.batchFriendlyId,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
if (batchRun.resumedAt) {
|
||||
logger.info("CreateCheckpointService: Batch already resumed", {
|
||||
batchRun,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
keepRunAlive: true,
|
||||
};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//sleep to test slow checkpoints
|
||||
// Sleep a random value between 4 and 30 seconds
|
||||
// await new Promise((resolve) => {
|
||||
// const waitSeconds = Math.floor(Math.random() * 26) + 4;
|
||||
// logger.log(`Sleep for ${waitSeconds} seconds`);
|
||||
// setTimeout(resolve, waitSeconds * 1000);
|
||||
// });
|
||||
|
||||
let metadata: string;
|
||||
|
||||
if (params.reason.type === "MANUAL") {
|
||||
metadata = JSON.stringify({
|
||||
...params.reason,
|
||||
attemptId: attempt.id,
|
||||
previousAttemptStatus: attempt.status,
|
||||
previousRunStatus: attempt.taskRun.status,
|
||||
} satisfies ManualCheckpointMetadata);
|
||||
} else {
|
||||
metadata = JSON.stringify(params.reason);
|
||||
}
|
||||
|
||||
const checkpoint = await this._prisma.checkpoint.create({
|
||||
data: {
|
||||
...CheckpointId.generate(),
|
||||
runtimeEnvironmentId: attempt.taskRun.runtimeEnvironmentId,
|
||||
projectId: attempt.taskRun.projectId,
|
||||
attemptId: attempt.id,
|
||||
attemptNumber: attempt.number,
|
||||
runId: attempt.taskRunId,
|
||||
location: params.location,
|
||||
type: params.docker ? "DOCKER" : "KUBERNETES",
|
||||
reason: params.reason.type,
|
||||
metadata,
|
||||
imageRef,
|
||||
},
|
||||
});
|
||||
|
||||
const eventService = new CreateCheckpointRestoreEventService(this._prisma);
|
||||
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: {
|
||||
id: attempt.id,
|
||||
},
|
||||
data: {
|
||||
status: params.reason.type === "RETRYING_AFTER_FAILURE" ? undefined : "PAUSED",
|
||||
taskRun: {
|
||||
update: {
|
||||
status: "WAITING_TO_RESUME",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
let checkpointEvent: CheckpointRestoreEvent | undefined;
|
||||
|
||||
switch (reason.type) {
|
||||
case "MANUAL":
|
||||
case "WAIT_FOR_DURATION": {
|
||||
let restoreAtUnixTimeMs: number;
|
||||
|
||||
if (reason.type === "MANUAL") {
|
||||
// Restore immediately if not specified, useful for live migration
|
||||
restoreAtUnixTimeMs = reason.restoreAtUnixTimeMs ?? Date.now();
|
||||
} else {
|
||||
restoreAtUnixTimeMs = reason.now + reason.ms;
|
||||
}
|
||||
|
||||
checkpointEvent = await eventService.checkpoint({
|
||||
checkpointId: checkpoint.id,
|
||||
});
|
||||
|
||||
if (checkpointEvent) {
|
||||
await marqs.requeueMessage(
|
||||
attempt.taskRunId,
|
||||
{
|
||||
type: "RESUME_AFTER_DURATION",
|
||||
resumableAttemptId: attempt.id,
|
||||
checkpointEventId: checkpointEvent.id,
|
||||
},
|
||||
restoreAtUnixTimeMs,
|
||||
"resume"
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_TASK": {
|
||||
checkpointEvent = await eventService.checkpoint({
|
||||
checkpointId: checkpoint.id,
|
||||
dependencyFriendlyRunId: reason.friendlyId,
|
||||
});
|
||||
|
||||
if (checkpointEvent) {
|
||||
//heartbeats will start again when the run resumes
|
||||
logger.log("CreateCheckpointService: Canceling heartbeat", {
|
||||
attemptId: attempt.id,
|
||||
taskRunId: attempt.taskRunId,
|
||||
type: "WAIT_FOR_TASK",
|
||||
reason,
|
||||
params,
|
||||
});
|
||||
await marqs?.cancelHeartbeat(attempt.taskRunId);
|
||||
|
||||
const childRun = await this._prisma.taskRun.findFirst({
|
||||
where: {
|
||||
friendlyId: reason.friendlyId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!childRun) {
|
||||
logger.error("CreateCheckpointService: WAIT_FOR_TASK child run not found", {
|
||||
friendlyId: reason.friendlyId,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
const resumeService = new ResumeDependentParentsService(this._prisma);
|
||||
const result = await resumeService.call({ id: childRun.id });
|
||||
|
||||
if (result.success) {
|
||||
logger.log("CreateCheckpointService: Resumed dependent parents", {
|
||||
result,
|
||||
childRun,
|
||||
attempt,
|
||||
checkpointEvent,
|
||||
params,
|
||||
});
|
||||
} else {
|
||||
logger.error("CreateCheckpointService: Failed to resume dependent parents", {
|
||||
result,
|
||||
childRun,
|
||||
attempt,
|
||||
checkpointEvent,
|
||||
params,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_BATCH": {
|
||||
checkpointEvent = await eventService.checkpoint({
|
||||
checkpointId: checkpoint.id,
|
||||
batchDependencyFriendlyId: reason.batchFriendlyId,
|
||||
});
|
||||
|
||||
if (checkpointEvent) {
|
||||
//heartbeats will start again when the run resumes
|
||||
logger.log("CreateCheckpointService: Canceling heartbeat", {
|
||||
attemptId: attempt.id,
|
||||
taskRunId: attempt.taskRunId,
|
||||
type: "WAIT_FOR_BATCH",
|
||||
params,
|
||||
});
|
||||
await marqs?.cancelHeartbeat(attempt.taskRunId);
|
||||
|
||||
// Routed by friendlyId; read the primary (this._prisma) so a just-resumed batch that still
|
||||
// lags the replica doesn't leave a stale resumedAt and suspend an already-resumed run.
|
||||
const batchRun = await this.runStore.findBatchTaskRunByFriendlyId(
|
||||
reason.batchFriendlyId,
|
||||
attempt.taskRun.runtimeEnvironmentId,
|
||||
undefined,
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!batchRun) {
|
||||
logger.error("CreateCheckpointService: Batch not found", {
|
||||
friendlyId: reason.batchFriendlyId,
|
||||
params,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
//if there's a message in the queue, we make sure the checkpoint event is on it
|
||||
await marqs.replaceMessage(attempt.taskRun.id, {
|
||||
checkpointEventId: checkpointEvent.id,
|
||||
});
|
||||
|
||||
await ResumeBatchRunService.enqueue(batchRun.id, batchRun.batchVersion === "v3");
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "RETRYING_AFTER_FAILURE": {
|
||||
checkpointEvent = await eventService.checkpoint({
|
||||
checkpointId: checkpoint.id,
|
||||
});
|
||||
|
||||
// ACK is already handled by attempt completion
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!checkpointEvent) {
|
||||
logger.error("No checkpoint event", {
|
||||
attemptId: attempt.id,
|
||||
checkpointId: checkpoint.id,
|
||||
params,
|
||||
});
|
||||
await marqs?.acknowledgeMessage(
|
||||
attempt.taskRunId,
|
||||
"No checkpoint event in CreateCheckpointService"
|
||||
);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
checkpoint,
|
||||
event: checkpointEvent,
|
||||
keepRunAlive: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { ManualCheckpointMetadata } from "@trigger.dev/core/v3";
|
||||
import type {
|
||||
Checkpoint,
|
||||
CheckpointRestoreEvent,
|
||||
CheckpointRestoreEventType,
|
||||
} from "@trigger.dev/database";
|
||||
import { isTaskRunAttemptStatus, isTaskRunStatus } from "~/database-types";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { safeJsonParse } from "~/utils/json";
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
interface CheckpointRestoreEventCallParams {
|
||||
checkpointId: string;
|
||||
type: CheckpointRestoreEventType;
|
||||
dependencyFriendlyRunId?: string;
|
||||
batchDependencyFriendlyId?: string;
|
||||
}
|
||||
|
||||
type CheckpointRestoreEventParams = Omit<CheckpointRestoreEventCallParams, "type">;
|
||||
|
||||
export class CreateCheckpointRestoreEventService extends BaseService {
|
||||
async checkpoint(params: CheckpointRestoreEventParams) {
|
||||
return this.#call({ ...params, type: "CHECKPOINT" });
|
||||
}
|
||||
|
||||
async restore(params: CheckpointRestoreEventParams) {
|
||||
return this.#call({ ...params, type: "RESTORE" });
|
||||
}
|
||||
|
||||
async #call(
|
||||
params: CheckpointRestoreEventCallParams
|
||||
): Promise<CheckpointRestoreEvent | undefined> {
|
||||
if (params.dependencyFriendlyRunId && params.batchDependencyFriendlyId) {
|
||||
logger.error("Only one dependency can be set", { params });
|
||||
return;
|
||||
}
|
||||
|
||||
const checkpoint = await this._prisma.checkpoint.findFirst({
|
||||
where: {
|
||||
id: params.checkpointId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!checkpoint) {
|
||||
logger.error("Checkpoint not found", { id: params.checkpointId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (params.type === "RESTORE" && checkpoint.reason === "MANUAL") {
|
||||
const manualRestoreSuccess = await this.#handleManualCheckpointRestore(checkpoint);
|
||||
if (!manualRestoreSuccess) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`Creating checkpoint/restore event`, { params });
|
||||
|
||||
let taskRunDependencyId: string | undefined;
|
||||
|
||||
if (params.dependencyFriendlyRunId) {
|
||||
const run = await this.runStore.findRun(
|
||||
{
|
||||
friendlyId: params.dependencyFriendlyRunId,
|
||||
},
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
dependency: {
|
||||
select: {
|
||||
id: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
taskRunDependencyId = run?.dependency?.id;
|
||||
|
||||
if (!taskRunDependencyId) {
|
||||
logger.error("Dependency or run not found", { runId: params.dependencyFriendlyRunId });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const checkpointEvent = await this._prisma.checkpointRestoreEvent.create({
|
||||
data: {
|
||||
checkpointId: checkpoint.id,
|
||||
runtimeEnvironmentId: checkpoint.runtimeEnvironmentId,
|
||||
projectId: checkpoint.projectId,
|
||||
attemptId: checkpoint.attemptId,
|
||||
runId: checkpoint.runId,
|
||||
type: params.type,
|
||||
reason: checkpoint.reason,
|
||||
metadata: checkpoint.metadata,
|
||||
...(taskRunDependencyId
|
||||
? {
|
||||
taskRunDependency: {
|
||||
connect: {
|
||||
id: taskRunDependencyId,
|
||||
},
|
||||
},
|
||||
}
|
||||
: undefined),
|
||||
...(params.batchDependencyFriendlyId
|
||||
? {
|
||||
batchTaskRunDependency: {
|
||||
connect: {
|
||||
friendlyId: params.batchDependencyFriendlyId,
|
||||
},
|
||||
},
|
||||
}
|
||||
: undefined),
|
||||
},
|
||||
});
|
||||
|
||||
return checkpointEvent;
|
||||
}
|
||||
|
||||
async #handleManualCheckpointRestore(checkpoint: Checkpoint): Promise<boolean> {
|
||||
const json = checkpoint.metadata ? safeJsonParse(checkpoint.metadata) : undefined;
|
||||
|
||||
// We need to restore the previous run and attempt status as saved in the metadata
|
||||
const metadata = ManualCheckpointMetadata.safeParse(json);
|
||||
|
||||
if (!metadata.success) {
|
||||
logger.error("Invalid metadata", { metadata });
|
||||
return false;
|
||||
}
|
||||
|
||||
const { attemptId, previousAttemptStatus, previousRunStatus } = metadata.data;
|
||||
|
||||
if (!isTaskRunAttemptStatus(previousAttemptStatus)) {
|
||||
logger.error("Invalid previous attempt status", { previousAttemptStatus });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!isTaskRunStatus(previousRunStatus)) {
|
||||
logger.error("Invalid previous run status", { previousRunStatus });
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const updatedAttempt = await this._prisma.taskRunAttempt.update({
|
||||
where: {
|
||||
id: attemptId,
|
||||
},
|
||||
data: {
|
||||
status: previousAttemptStatus,
|
||||
taskRun: {
|
||||
update: {
|
||||
data: {
|
||||
status: previousRunStatus,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
taskRun: {
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug("Set post resume statuses after manual checkpoint", {
|
||||
run: {
|
||||
id: updatedAttempt.taskRun.id,
|
||||
status: updatedAttempt.taskRun.status,
|
||||
},
|
||||
attempt: {
|
||||
id: updatedAttempt.id,
|
||||
status: updatedAttempt.status,
|
||||
},
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error("Failed to set post resume statuses", {
|
||||
error:
|
||||
error instanceof Error
|
||||
? {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
}
|
||||
: error,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import type { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import { tryCatch } from "@trigger.dev/core/v3";
|
||||
import type { BackgroundWorker, PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { syncTaskIdentifiers } from "~/services/taskIdentifierRegistry.server";
|
||||
import { type TaskMetadataCache } from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
||||
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import {
|
||||
createWorkerResources,
|
||||
stripBackgroundWorkerMetadataForStorage,
|
||||
syncDeclarativeSchedules,
|
||||
} from "./createBackgroundWorker.server";
|
||||
import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
|
||||
import { projectPubSub } from "./projectPubSub.server";
|
||||
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
||||
import { CURRENT_DEPLOYMENT_LABEL, BackgroundWorkerId } from "@trigger.dev/core/v3/isomorphic";
|
||||
|
||||
/**
|
||||
* This service was only used before the new build system was introduced in v3.
|
||||
* It's now replaced by the CreateDeploymentBackgroundWorkerServiceV4.
|
||||
*
|
||||
* @deprecated
|
||||
*/
|
||||
export class CreateDeploymentBackgroundWorkerServiceV3 extends BaseService {
|
||||
private readonly _taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
prisma?: PrismaClientOrTransaction,
|
||||
replica?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
super(prisma, replica);
|
||||
this._taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
public async call(
|
||||
projectRef: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
deploymentId: string,
|
||||
body: CreateBackgroundWorkerRequestBody
|
||||
): Promise<BackgroundWorker | undefined> {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
span.setAttribute("projectRef", projectRef);
|
||||
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId: deploymentId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (deployment.status !== "DEPLOYING") {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.create({
|
||||
data: {
|
||||
...BackgroundWorkerId.generate(),
|
||||
version: deployment.version,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
metadata: stripBackgroundWorkerMetadataForStorage(body.metadata),
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
supportsLazyAttempts: body.supportsLazyAttempts,
|
||||
engine: body.engine,
|
||||
},
|
||||
});
|
||||
|
||||
//upgrade the project to engine "V2" if it's not already
|
||||
if (environment.project.engine === "V1" && body.engine === "V2") {
|
||||
await this._prisma.project.update({
|
||||
where: {
|
||||
id: environment.project.id,
|
||||
},
|
||||
data: {
|
||||
engine: "V2",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
let workerTaskEntries: Awaited<ReturnType<typeof createWorkerResources>> = [];
|
||||
try {
|
||||
workerTaskEntries = await createWorkerResources(
|
||||
body.metadata,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma
|
||||
);
|
||||
await syncDeclarativeSchedules(
|
||||
body.metadata.tasks,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma
|
||||
);
|
||||
} catch (error) {
|
||||
const name = error instanceof Error ? error.name : "UnknownError";
|
||||
const message = error instanceof Error ? error.message : JSON.stringify(error);
|
||||
|
||||
await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
failedAt: new Date(),
|
||||
errorData: {
|
||||
name,
|
||||
message,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Link the deployment with the background worker
|
||||
await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
status: "DEPLOYED",
|
||||
workerId: backgroundWorker.id,
|
||||
deployedAt: new Date(),
|
||||
type: backgroundWorker.engine === "V2" ? "MANAGED" : "V1",
|
||||
},
|
||||
});
|
||||
|
||||
//set this deployment as the current deployment for this environment
|
||||
await this._prisma.workerDeploymentPromotion.upsert({
|
||||
where: {
|
||||
environmentId_label: {
|
||||
environmentId: environment.id,
|
||||
label: CURRENT_DEPLOYMENT_LABEL,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
deploymentId: deployment.id,
|
||||
environmentId: environment.id,
|
||||
label: CURRENT_DEPLOYMENT_LABEL,
|
||||
},
|
||||
update: {
|
||||
deploymentId: deployment.id,
|
||||
},
|
||||
});
|
||||
|
||||
const [syncIdError] = await tryCatch(
|
||||
syncTaskIdentifiers(
|
||||
environment.id,
|
||||
environment.projectId,
|
||||
backgroundWorker.id,
|
||||
body.metadata.tasks.map((t) => ({ id: t.id, triggerSource: t.triggerSource }))
|
||||
)
|
||||
);
|
||||
|
||||
if (syncIdError) {
|
||||
logger.error("Error syncing task identifiers", { error: syncIdError });
|
||||
}
|
||||
|
||||
// V3 promotes the deployment immediately above, so this worker is now
|
||||
// current for the env — write both keyspaces atomically. Cache calls
|
||||
// log+swallow internally. Empty `workerTaskEntries` is intentional: the
|
||||
// populate methods clear stale hashes for zero-task deploys.
|
||||
await this._taskMetaCache.populateByCurrentWorker(
|
||||
environment.id,
|
||||
backgroundWorker.id,
|
||||
workerTaskEntries
|
||||
);
|
||||
|
||||
try {
|
||||
//send a notification that a new worker has been created
|
||||
await projectPubSub.publish(
|
||||
`project:${environment.projectId}:env:${environment.id}`,
|
||||
"WORKER_CREATED",
|
||||
{
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
createdAt: backgroundWorker.createdAt,
|
||||
taskCount: body.metadata.tasks.length,
|
||||
type: "deployed",
|
||||
}
|
||||
);
|
||||
await updateEnvConcurrencyLimits(environment);
|
||||
} catch (err) {
|
||||
logger.error("Failed to publish WORKER_CREATED event", { err });
|
||||
}
|
||||
|
||||
if (deployment.imageReference) {
|
||||
socketIo.providerNamespace.emit("PRE_PULL_DEPLOYMENT", {
|
||||
version: "v1",
|
||||
imageRef: deployment.imageReference,
|
||||
shortCode: deployment.shortCode,
|
||||
// identifiers
|
||||
deploymentId: deployment.id,
|
||||
envId: environment.id,
|
||||
envType: environment.type,
|
||||
orgId: environment.organizationId,
|
||||
projectId: deployment.projectId,
|
||||
});
|
||||
}
|
||||
|
||||
await ExecuteTasksWaitingForDeployService.enqueue(backgroundWorker.id);
|
||||
await PerformDeploymentAlertsService.enqueue(deployment.id);
|
||||
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
|
||||
|
||||
return backgroundWorker;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
import type { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import { logger, tryCatch } from "@trigger.dev/core/v3";
|
||||
import type {
|
||||
BackgroundWorker,
|
||||
PrismaClientOrTransaction,
|
||||
WorkerDeployment,
|
||||
} from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { type TaskMetadataCache } from "~/services/taskMetadataCache.server";
|
||||
import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import {
|
||||
createBackgroundFiles,
|
||||
createWorkerResources,
|
||||
syncDeclarativeSchedules,
|
||||
} from "./createBackgroundWorker.server";
|
||||
import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server";
|
||||
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
||||
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
|
||||
import { env } from "~/env.server";
|
||||
|
||||
export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
|
||||
private readonly _taskMetaCache: TaskMetadataCache;
|
||||
|
||||
constructor(
|
||||
prisma?: PrismaClientOrTransaction,
|
||||
replica?: PrismaClientOrTransaction,
|
||||
taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance
|
||||
) {
|
||||
super(prisma, replica);
|
||||
this._taskMetaCache = taskMetaCache;
|
||||
}
|
||||
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
deploymentId: string,
|
||||
body: CreateBackgroundWorkerRequestBody
|
||||
): Promise<BackgroundWorker | undefined> {
|
||||
return this.traceWithEnv("call", environment, async (span) => {
|
||||
span.setAttribute("deploymentId", deploymentId);
|
||||
|
||||
const { buildPlatform, targetPlatform } = body;
|
||||
|
||||
if (buildPlatform) {
|
||||
span.setAttribute("buildPlatform", buildPlatform);
|
||||
}
|
||||
if (targetPlatform) {
|
||||
span.setAttribute("targetPlatform", targetPlatform);
|
||||
}
|
||||
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId: deploymentId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.warn("createDeploymentBackgroundWorker: deployment not found", {
|
||||
deploymentId,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle multi-platform builds
|
||||
const deploymentPlatforms = deployment.imagePlatform?.split(",") ?? [];
|
||||
if (deploymentPlatforms.length > 1) {
|
||||
span.setAttribute("deploymentPlatforms", deploymentPlatforms.join(","));
|
||||
|
||||
// We will only create a background worker for the first platform
|
||||
const firstPlatform = deploymentPlatforms[0];
|
||||
|
||||
if (targetPlatform && firstPlatform !== targetPlatform) {
|
||||
throw new ServiceValidationError(
|
||||
`Ignoring target platform ${targetPlatform} for multi-platform deployment ${deployment.imagePlatform}`,
|
||||
400
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Late-retry idempotency: if a worker was registered by a prior fully-
|
||||
// successful attempt and the deployment already moved past BUILDING, return
|
||||
// that worker so the CLI can finalize instead of seeing a 5xx.
|
||||
if (deployment.workerId) {
|
||||
const linkedWorker = await this._prisma.backgroundWorker.findFirst({
|
||||
where: { id: deployment.workerId },
|
||||
});
|
||||
if (linkedWorker) {
|
||||
return linkedWorker;
|
||||
}
|
||||
}
|
||||
|
||||
if (deployment.status !== "BUILDING") {
|
||||
logger.warn("createDeploymentBackgroundWorker: deployment not in BUILDING state", {
|
||||
deploymentId,
|
||||
deploymentStatus: deployment.status,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const [findOrCreateError, backgroundWorker] = await tryCatch(
|
||||
findOrCreateBackgroundWorker(environment, deployment, body, this._prisma)
|
||||
);
|
||||
|
||||
if (findOrCreateError) {
|
||||
// Definitive failures (e.g. contentHash drift) surface as
|
||||
// `ServiceValidationError` — fail the deployment so the operator sees it
|
||||
// immediately instead of waiting 8 minutes for the timeout. Transient
|
||||
// races throw a plain `Error` and propagate as 5xx without failing.
|
||||
if (findOrCreateError instanceof ServiceValidationError) {
|
||||
// `#failBackgroundWorkerDeployment` already throws its argument; the
|
||||
// outer `throw` covers the non-SVE branch.
|
||||
await this.#failBackgroundWorkerDeployment(deployment, findOrCreateError, environment);
|
||||
}
|
||||
throw findOrCreateError;
|
||||
}
|
||||
|
||||
//upgrade the project to engine "V2" if it's not already
|
||||
if (environment.project.engine === "V1" && body.engine === "V2") {
|
||||
await this._prisma.project.update({
|
||||
where: {
|
||||
id: environment.project.id,
|
||||
},
|
||||
data: {
|
||||
engine: "V2",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const [filesError, tasksToBackgroundFiles] = await tryCatch(
|
||||
createBackgroundFiles(
|
||||
body.metadata.sourceFiles,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma
|
||||
)
|
||||
);
|
||||
|
||||
if (filesError) {
|
||||
logger.error("Error creating background worker files", {
|
||||
error: filesError,
|
||||
});
|
||||
|
||||
const serviceError = new ServiceValidationError("Error creating background worker files");
|
||||
|
||||
await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
|
||||
|
||||
throw serviceError;
|
||||
}
|
||||
|
||||
const [resourcesError, workerTaskEntries] = await tryCatch(
|
||||
createWorkerResources(
|
||||
body.metadata,
|
||||
backgroundWorker,
|
||||
environment,
|
||||
this._prisma,
|
||||
tasksToBackgroundFiles
|
||||
)
|
||||
);
|
||||
|
||||
if (resourcesError) {
|
||||
if (resourcesError instanceof ServiceValidationError) {
|
||||
// Customer-facing config error (e.g. duplicate task ids). Surface the
|
||||
// real message to the client via the rethrow.
|
||||
logger.warn("Error creating background worker resources", {
|
||||
error: resourcesError.message,
|
||||
});
|
||||
|
||||
await this.#failBackgroundWorkerDeployment(deployment, resourcesError, environment);
|
||||
throw resourcesError;
|
||||
}
|
||||
|
||||
logger.error("Error creating background worker resources", {
|
||||
error: resourcesError,
|
||||
});
|
||||
|
||||
const serviceError = new ServiceValidationError(
|
||||
"Error creating background worker resources"
|
||||
);
|
||||
|
||||
await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
|
||||
|
||||
throw serviceError;
|
||||
}
|
||||
|
||||
// V4 build path: worker created but NOT yet promoted to current. Write
|
||||
// only the `task-meta:by-worker:{workerId}` keyspace so locked-version
|
||||
// triggers against this build hit the cache. Promotion (which writes the
|
||||
// env keyspace) happens later via finalizeDeployment → changeCurrentDeployment.
|
||||
// Cache calls log+swallow internally, so a Redis blip can't stall the
|
||||
// deployment state machine. Empty entries clears stale hashes.
|
||||
if (workerTaskEntries) {
|
||||
await this._taskMetaCache.populateByWorker(backgroundWorker.id, workerTaskEntries);
|
||||
}
|
||||
|
||||
const [schedulesError] = await tryCatch(
|
||||
syncDeclarativeSchedules(body.metadata.tasks, backgroundWorker, environment, this._prisma)
|
||||
);
|
||||
|
||||
if (schedulesError) {
|
||||
if (schedulesError instanceof ServiceValidationError) {
|
||||
// Customer schedule config (typically invalid cron). Surface to
|
||||
// client via the rethrow; system returns gracefully.
|
||||
logger.warn("Error syncing declarative schedules", {
|
||||
error: schedulesError.message,
|
||||
});
|
||||
|
||||
await this.#failBackgroundWorkerDeployment(deployment, schedulesError, environment);
|
||||
throw schedulesError;
|
||||
}
|
||||
|
||||
// Wrapping the underlying error into a ServiceValidationError below
|
||||
// would otherwise hide it once the SDK-level filter drops SVEs; log at
|
||||
// error so the underlying cause stays visible. Mirrors the
|
||||
// waitpointCompletionPacket.server.ts pattern from dac9c83bd.
|
||||
logger.error("Error syncing declarative schedules", {
|
||||
error: schedulesError,
|
||||
});
|
||||
|
||||
const serviceError = new ServiceValidationError("Error syncing declarative schedules");
|
||||
|
||||
await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
|
||||
|
||||
throw serviceError;
|
||||
}
|
||||
|
||||
// Guarded BUILDING → DEPLOYING transition. `updateMany` for optimistic concurrency control
|
||||
const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
status: "BUILDING",
|
||||
},
|
||||
data: {
|
||||
status: "DEPLOYING",
|
||||
workerId: backgroundWorker.id,
|
||||
builtAt: new Date(),
|
||||
type: backgroundWorker.engine === "V2" ? "MANAGED" : "V1",
|
||||
// runtime is already set when the deployment is created, we only need to set the version
|
||||
runtimeVersion: body.metadata.runtimeVersion,
|
||||
},
|
||||
});
|
||||
|
||||
if (updatedCount === 0) {
|
||||
logger.warn(
|
||||
"createDeploymentBackgroundWorker: deployment no longer in BUILDING state, skipping DEPLOYING transition",
|
||||
{
|
||||
deploymentId,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
}
|
||||
);
|
||||
return backgroundWorker;
|
||||
}
|
||||
|
||||
await TimeoutDeploymentService.enqueue(
|
||||
deployment.id,
|
||||
"DEPLOYING",
|
||||
"Indexing timed out",
|
||||
new Date(Date.now() + env.DEPLOY_TIMEOUT_MS)
|
||||
);
|
||||
|
||||
return backgroundWorker;
|
||||
});
|
||||
}
|
||||
|
||||
async #failBackgroundWorkerDeployment(
|
||||
deployment: WorkerDeployment,
|
||||
error: Error,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
// Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING
|
||||
// transition in `call()`. With idempotent retries, two attempts can run side-by-side;
|
||||
// without the predicate, one attempt's failure could downgrade the deployment after
|
||||
// the other already flipped it to DEPLOYING, leaving it stuck in FAILED with a worker.
|
||||
const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
status: "BUILDING",
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
failedAt: new Date(),
|
||||
errorData: {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (updatedCount === 0) {
|
||||
logger.warn(
|
||||
"failBackgroundWorkerDeployment: deployment moved out of BUILDING during call, skipping FAILED transition",
|
||||
{
|
||||
deploymentId: deployment.id,
|
||||
originalError: error.message,
|
||||
}
|
||||
);
|
||||
} else {
|
||||
// Only dequeue the timeout if we actually flipped to FAILED — otherwise a
|
||||
// sibling attempt may have just enqueued it as part of a successful
|
||||
// BUILDING → DEPLOYING transition.
|
||||
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
|
||||
|
||||
recordDeploymentOutcome({
|
||||
status: "FAILED",
|
||||
deploymentFriendlyId: deployment.friendlyId,
|
||||
organizationId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
reason: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
import type { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
|
||||
import { BackgroundWorkerId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
isUniqueConstraintError,
|
||||
type BackgroundWorker,
|
||||
type PrismaClientOrTransaction,
|
||||
type WorkerDeployment,
|
||||
} from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { ServiceValidationError } from "../common.server";
|
||||
import { stripBackgroundWorkerMetadataForStorage } from "../stripBackgroundWorkerMetadataForStorage.server";
|
||||
|
||||
/**
|
||||
* Idempotent on `(project, environment, version)` for sequential calls, not concurrent calls.
|
||||
*
|
||||
* Failure shapes the caller distinguishes:
|
||||
* - `ServiceValidationError` (409): definitive — contentHash drift means a different build
|
||||
* is being pushed under the same deployment version, so the caller should fail the
|
||||
* deployment instead of waiting for a timeout.
|
||||
* - Plain `Error`: transient — two attempts raced the `create()` call and the loser caught
|
||||
* the unique-index violation. The caller should propagate this as 5xx so the CLI's
|
||||
* retry/backoff hits findFirst on the next attempt and returns the winner's row.
|
||||
*/
|
||||
export async function findOrCreateBackgroundWorker(
|
||||
environment: AuthenticatedEnvironment,
|
||||
deployment: WorkerDeployment,
|
||||
body: CreateBackgroundWorkerRequestBody,
|
||||
prisma: PrismaClientOrTransaction
|
||||
): Promise<BackgroundWorker> {
|
||||
const existing = await prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
projectId: environment.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
version: deployment.version,
|
||||
},
|
||||
});
|
||||
|
||||
if (existing && existing.contentHash === body.metadata.contentHash) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (existing) {
|
||||
throw new ServiceValidationError(
|
||||
"A background worker for this deployment version already exists with a different content hash",
|
||||
409
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await prisma.backgroundWorker.create({
|
||||
data: {
|
||||
...BackgroundWorkerId.generate(),
|
||||
version: deployment.version,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
metadata: stripBackgroundWorkerMetadataForStorage(body.metadata),
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
supportsLazyAttempts: body.supportsLazyAttempts,
|
||||
engine: body.engine,
|
||||
runtime: body.metadata.runtime,
|
||||
runtimeVersion: body.metadata.runtimeVersion,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
// Concurrent attempts raced past `findFirst` and both reached `create`. Surface
|
||||
// a clear, non-Prisma error so the 5xx the caller returns isn't an opaque
|
||||
// P2002 — the CLI's retry will then hit `findFirst` and find the winner's row.
|
||||
// Intentionally NOT a ServiceValidationError so the caller doesn't fail-deploy
|
||||
// on a transient race.
|
||||
if (isUniqueConstraintError(error, ["projectId", "runtimeEnvironmentId", "version"])) {
|
||||
throw new Error(
|
||||
"Concurrent background worker registration detected for this deployment version; please retry"
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { OrganizationIntegration } from "@trigger.dev/database";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { OrgIntegrationRepository } from "~/models/orgIntegration.server";
|
||||
|
||||
export class CreateOrgIntegrationService extends BaseService {
|
||||
public async call(
|
||||
userId: string,
|
||||
orgId: string,
|
||||
serviceName: string,
|
||||
code: string
|
||||
): Promise<OrganizationIntegration | undefined> {
|
||||
// Get the org
|
||||
const org = await this._prisma.organization.findUnique({
|
||||
where: {
|
||||
id: orgId,
|
||||
members: {
|
||||
some: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!org) {
|
||||
throw new Error("Organization not found");
|
||||
}
|
||||
|
||||
return OrgIntegrationRepository.createOrgIntegration(serviceName, code, org);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
import type { V3TaskRunExecution } from "@trigger.dev/core/v3";
|
||||
import { parsePacket } from "@trigger.dev/core/v3";
|
||||
import type { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
|
||||
import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { $transaction, prisma } from "~/db.server";
|
||||
import { findQueueInEnvironment } from "~/models/taskQueue.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { reportInvocationUsage } from "~/services/platform.v3.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { machinePresetFromConfig, machinePresetFromRun } from "../machinePresets.server";
|
||||
import { FINAL_RUN_STATUSES } from "../taskStatus";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { CrashTaskRunService } from "./crashTaskRun.server";
|
||||
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
|
||||
import { runStore } from "../runStore.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
|
||||
export class CreateTaskRunAttemptService extends BaseService {
|
||||
public async call({
|
||||
runId,
|
||||
authenticatedEnv,
|
||||
setToExecuting = true,
|
||||
startAtZero = false,
|
||||
}: {
|
||||
runId: string;
|
||||
authenticatedEnv?: AuthenticatedEnvironment;
|
||||
setToExecuting?: boolean;
|
||||
startAtZero?: boolean;
|
||||
}): Promise<{
|
||||
execution: V3TaskRunExecution;
|
||||
run: TaskRun;
|
||||
attempt: TaskRunAttempt;
|
||||
}> {
|
||||
const environment =
|
||||
authenticatedEnv ?? (await getAuthenticatedEnvironmentFromRun(runId, this._prisma));
|
||||
|
||||
if (!environment) {
|
||||
throw new ServiceValidationError("Environment not found", 404);
|
||||
}
|
||||
|
||||
const isFriendlyId = runId.startsWith("run_");
|
||||
|
||||
return await this.traceWithEnv("call()", environment, async (span) => {
|
||||
if (isFriendlyId) {
|
||||
span.setAttribute("taskRunFriendlyId", runId);
|
||||
} else {
|
||||
span.setAttribute("taskRunId", runId);
|
||||
}
|
||||
|
||||
const taskRun = await this.runStore.findRun(
|
||||
{
|
||||
id: !isFriendlyId ? runId : undefined,
|
||||
friendlyId: isFriendlyId ? runId : undefined,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
{
|
||||
include: {
|
||||
attempts: {
|
||||
take: 1,
|
||||
orderBy: {
|
||||
number: "desc",
|
||||
},
|
||||
},
|
||||
batchItems: {
|
||||
include: {
|
||||
batchTaskRun: {
|
||||
select: {
|
||||
friendlyId: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
logger.debug("Creating a task run attempt", { taskRun });
|
||||
|
||||
if (!taskRun) {
|
||||
throw new ServiceValidationError("Task run not found", 404);
|
||||
}
|
||||
|
||||
span.setAttribute("taskRunId", taskRun.id);
|
||||
span.setAttribute("taskRunFriendlyId", taskRun.friendlyId);
|
||||
span.setAttribute("taskRunStatus", taskRun.status);
|
||||
|
||||
if (taskRun.status === "CANCELED") {
|
||||
throw new ServiceValidationError("Task run is cancelled", 400);
|
||||
}
|
||||
|
||||
// If the run is finalized, it's pointless to create another attempt
|
||||
if (FINAL_RUN_STATUSES.includes(taskRun.status)) {
|
||||
throw new ServiceValidationError("Task run is already finished", 400);
|
||||
}
|
||||
|
||||
const lockedWorker = await controlPlaneResolver.resolveRunLockedWorker({
|
||||
lockedById: taskRun.lockedById,
|
||||
});
|
||||
const lockedBy = lockedWorker?.lockedBy;
|
||||
|
||||
if (!lockedBy) {
|
||||
throw new ServiceValidationError("Task run is not locked", 400);
|
||||
}
|
||||
|
||||
const queue = await findQueueInEnvironment(taskRun.queue, environment.id, lockedBy.id);
|
||||
|
||||
if (!queue) {
|
||||
throw new ServiceValidationError("Queue not found", 404);
|
||||
}
|
||||
|
||||
const nextAttemptNumber = taskRun.attempts[0]
|
||||
? taskRun.attempts[0].number + 1
|
||||
: startAtZero
|
||||
? 0
|
||||
: 1;
|
||||
|
||||
if (nextAttemptNumber > MAX_TASK_RUN_ATTEMPTS) {
|
||||
const service = new CrashTaskRunService(this._prisma);
|
||||
await service.call(taskRun.id, {
|
||||
reason: lockedBy.worker.supportsLazyAttempts
|
||||
? "Max attempts reached."
|
||||
: "Max attempts reached. Please upgrade your CLI and SDK.",
|
||||
});
|
||||
|
||||
throw new ServiceValidationError("Max attempts reached", 400);
|
||||
}
|
||||
|
||||
const taskRunAttempt = await $transaction(this._prisma, "create attempt", async (tx) => {
|
||||
const taskRunAttempt = await tx.taskRunAttempt.create({
|
||||
data: {
|
||||
number: nextAttemptNumber,
|
||||
friendlyId: generateFriendlyId("attempt"),
|
||||
taskRunId: taskRun.id,
|
||||
startedAt: new Date(),
|
||||
backgroundWorkerId: lockedBy.worker.id,
|
||||
backgroundWorkerTaskId: lockedBy.id,
|
||||
status: setToExecuting ? "EXECUTING" : "PENDING",
|
||||
queueId: queue.id,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
});
|
||||
|
||||
await tx.taskRun.update({
|
||||
where: {
|
||||
id: taskRun.id,
|
||||
},
|
||||
data: {
|
||||
status: setToExecuting ? "EXECUTING" : undefined,
|
||||
executedAt: taskRun.executedAt ?? new Date(),
|
||||
attemptNumber: nextAttemptNumber,
|
||||
},
|
||||
});
|
||||
|
||||
if (taskRun.ttl) {
|
||||
await ExpireEnqueuedRunService.ack(taskRun.id, tx);
|
||||
}
|
||||
|
||||
return taskRunAttempt;
|
||||
});
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
logger.error("Failed to create task run attempt", { runId: taskRun.id, nextAttemptNumber });
|
||||
throw new ServiceValidationError("Failed to create task run attempt", 500);
|
||||
}
|
||||
|
||||
if (taskRunAttempt.number === 1 && taskRun.baseCostInCents > 0) {
|
||||
await reportInvocationUsage(environment.organizationId, taskRun.baseCostInCents, {
|
||||
runId: taskRun.id,
|
||||
});
|
||||
}
|
||||
|
||||
const machinePreset =
|
||||
machinePresetFromRun(taskRun) ?? machinePresetFromConfig(lockedBy.machineConfig ?? {});
|
||||
|
||||
const metadata = await parsePacket({
|
||||
data: taskRun.metadata ?? undefined,
|
||||
dataType: taskRun.metadataType,
|
||||
});
|
||||
|
||||
const execution: V3TaskRunExecution = {
|
||||
task: {
|
||||
id: lockedBy.slug,
|
||||
filePath: lockedBy.filePath,
|
||||
exportName: lockedBy.exportName ?? "@deprecated",
|
||||
},
|
||||
attempt: {
|
||||
id: taskRunAttempt.friendlyId,
|
||||
number: taskRunAttempt.number,
|
||||
startedAt: taskRunAttempt.startedAt ?? taskRunAttempt.createdAt,
|
||||
backgroundWorkerId: lockedBy.worker.id,
|
||||
backgroundWorkerTaskId: lockedBy.id,
|
||||
status: "EXECUTING" as const,
|
||||
},
|
||||
run: {
|
||||
id: taskRun.friendlyId,
|
||||
payload: taskRun.payload,
|
||||
payloadType: taskRun.payloadType,
|
||||
context: taskRun.context,
|
||||
createdAt: taskRun.createdAt,
|
||||
tags: taskRun.runTags ?? [],
|
||||
isTest: taskRun.isTest,
|
||||
isReplay: !!taskRun.replayedFromTaskRunFriendlyId,
|
||||
idempotencyKey: taskRun.idempotencyKey ?? undefined,
|
||||
startedAt: taskRun.startedAt ?? taskRun.createdAt,
|
||||
durationMs: taskRun.usageDurationMs,
|
||||
costInCents: taskRun.costInCents,
|
||||
baseCostInCents: taskRun.baseCostInCents,
|
||||
maxAttempts: taskRun.maxAttempts ?? undefined,
|
||||
version: lockedBy.worker.version,
|
||||
metadata,
|
||||
maxDuration: taskRun.maxDurationInSeconds ?? undefined,
|
||||
},
|
||||
queue: {
|
||||
id: queue.friendlyId,
|
||||
name: queue.name,
|
||||
},
|
||||
environment: {
|
||||
id: environment.id,
|
||||
slug: environment.slug,
|
||||
type: environment.type,
|
||||
},
|
||||
organization: {
|
||||
id: environment.organization.id,
|
||||
slug: environment.organization.slug,
|
||||
name: environment.organization.title,
|
||||
},
|
||||
project: {
|
||||
id: environment.project.id,
|
||||
ref: environment.project.externalRef,
|
||||
slug: environment.project.slug,
|
||||
name: environment.project.name,
|
||||
},
|
||||
batch:
|
||||
taskRun.batchItems[0] && taskRun.batchItems[0].batchTaskRun
|
||||
? { id: taskRun.batchItems[0].batchTaskRun.friendlyId }
|
||||
: undefined,
|
||||
machine: machinePreset,
|
||||
};
|
||||
|
||||
return {
|
||||
execution,
|
||||
run: taskRun,
|
||||
attempt: taskRunAttempt,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function getAuthenticatedEnvironmentFromRun(
|
||||
friendlyId: string,
|
||||
prismaClient?: PrismaClientOrTransaction
|
||||
) {
|
||||
const isFriendlyId = friendlyId.startsWith("run_");
|
||||
|
||||
const taskRun = await runStore.findRun(
|
||||
{
|
||||
id: !isFriendlyId ? friendlyId : undefined,
|
||||
friendlyId: isFriendlyId ? friendlyId : undefined,
|
||||
},
|
||||
{
|
||||
select: {
|
||||
runtimeEnvironmentId: true,
|
||||
},
|
||||
},
|
||||
prismaClient ?? prisma
|
||||
);
|
||||
|
||||
if (!taskRun) {
|
||||
return;
|
||||
}
|
||||
|
||||
return (
|
||||
(await controlPlaneResolver.resolveAuthenticatedEnv(taskRun.runtimeEnvironmentId)) ?? undefined
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
|
||||
export class DeleteTaskRunTemplateService extends BaseService {
|
||||
public async call(environment: AuthenticatedEnvironment, templateId: string) {
|
||||
try {
|
||||
await this._prisma.taskRunTemplate.delete({
|
||||
where: {
|
||||
id: templateId,
|
||||
projectId: environment.projectId,
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Error deleting template: ${e instanceof Error ? e.message : JSON.stringify(e)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
type Options = {
|
||||
projectId: string;
|
||||
userId: string;
|
||||
friendlyId: string;
|
||||
};
|
||||
|
||||
export class DeleteTaskScheduleService extends BaseService {
|
||||
public async call({ projectId, userId, friendlyId }: Options) {
|
||||
//first check that the user has access to the project
|
||||
const project = await this._prisma.project.findFirst({
|
||||
where: {
|
||||
id: projectId,
|
||||
organization: {
|
||||
members: {
|
||||
some: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!project) {
|
||||
throw new Error("User does not have access to the project");
|
||||
}
|
||||
|
||||
try {
|
||||
const schedule = await this._prisma.taskSchedule.findFirst({
|
||||
where: {
|
||||
friendlyId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!schedule) {
|
||||
throw new Error("Schedule not found");
|
||||
}
|
||||
|
||||
if (schedule.type === "DECLARATIVE") {
|
||||
throw new Error("Cannot delete declarative schedules");
|
||||
}
|
||||
|
||||
await this._prisma.taskSchedule.delete({
|
||||
where: {
|
||||
id: schedule.id,
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Error deleting schedule: ${e instanceof Error ? e.message : JSON.stringify(e)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { Prisma } from "@trigger.dev/database";
|
||||
|
||||
/**
|
||||
* Where-clause for resolving a dependent/parent TaskRunAttempt by friendlyId,
|
||||
* scoped to the caller's environment via the related run. The env scope keeps a
|
||||
* foreign friendlyId from resolving onto the new batch. Standalone builder so
|
||||
* the scope can be asserted directly in tests.
|
||||
*/
|
||||
export function dependentAttemptWhere(
|
||||
friendlyId: string,
|
||||
environmentId: string
|
||||
): Prisma.TaskRunAttemptWhereInput {
|
||||
return {
|
||||
friendlyId,
|
||||
taskRun: { runtimeEnvironmentId: environmentId },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow";
|
||||
import { type WorkerDeployment, type Project } from "@trigger.dev/database";
|
||||
import {
|
||||
BuildServerMetadata,
|
||||
logger,
|
||||
type GitMeta,
|
||||
type DeploymentEvent,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
||||
import { env } from "~/env.server";
|
||||
import { createRemoteImageBuild } from "../remoteImageBuilder.server";
|
||||
import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server";
|
||||
import { enqueueBuild, generateRegistryCredentials } from "~/services/platform.v3.server";
|
||||
import { AppendInput, AppendRecord, S2 } from "@s2-dev/streamstore";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
|
||||
const S2_TOKEN_KEY_PREFIX = "s2-token:read:deployment-event-stream:project:";
|
||||
const s2TokenRedis = createRedisClient("s2-token-cache", {
|
||||
host: env.CACHE_REDIS_HOST,
|
||||
port: env.CACHE_REDIS_PORT,
|
||||
username: env.CACHE_REDIS_USERNAME,
|
||||
password: env.CACHE_REDIS_PASSWORD,
|
||||
tlsDisabled: env.CACHE_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.CACHE_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
});
|
||||
const s2 = env.S2_ENABLED === "1" ? new S2({ accessToken: env.S2_ACCESS_TOKEN }) : undefined;
|
||||
|
||||
export class DeploymentService extends BaseService {
|
||||
/**
|
||||
* Progresses a deployment from PENDING to INSTALLING and then to BUILDING.
|
||||
* Also extends the deployment timeout.
|
||||
*
|
||||
* When progressing to BUILDING, the remote Depot build is also created.
|
||||
*
|
||||
* Only acts when the current status allows. Not idempotent.
|
||||
*
|
||||
* @param authenticatedEnv The environment which the deployment belongs to.
|
||||
* @param friendlyId The friendly deployment ID.
|
||||
* @param updates Optional deployment details to persist.
|
||||
*/
|
||||
public progressDeployment(
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
friendlyId: string,
|
||||
updates: Partial<
|
||||
Pick<WorkerDeployment, "contentHash" | "runtime"> & {
|
||||
git: GitMeta;
|
||||
buildServerMetadata: BuildServerMetadata;
|
||||
}
|
||||
>
|
||||
) {
|
||||
const { buildServerMetadata: newBuildServerMetadata, ...restUpdates } = updates;
|
||||
|
||||
const validateDeployment = (
|
||||
deployment: Pick<WorkerDeployment, "id" | "status"> & {
|
||||
buildServerMetadata?: BuildServerMetadata;
|
||||
}
|
||||
) => {
|
||||
if (deployment.status !== "PENDING" && deployment.status !== "INSTALLING") {
|
||||
logger.warn(
|
||||
"Attempted progressing deployment that is not in PENDING or INSTALLING status",
|
||||
{
|
||||
deployment,
|
||||
}
|
||||
);
|
||||
return errAsync({ type: "deployment_cannot_be_progressed" as const });
|
||||
}
|
||||
|
||||
return okAsync(deployment);
|
||||
};
|
||||
|
||||
const progressToInstalling = (
|
||||
deployment: Pick<WorkerDeployment, "id"> & { buildServerMetadata?: BuildServerMetadata }
|
||||
) => {
|
||||
const existingBuildServerMetadata = deployment.buildServerMetadata;
|
||||
|
||||
return fromPromise(
|
||||
this._prisma.workerDeployment.updateMany({
|
||||
where: { id: deployment.id, status: "PENDING" }, // status could've changed in the meantime, we're not locking the row
|
||||
data: {
|
||||
...restUpdates,
|
||||
...(newBuildServerMetadata
|
||||
? {
|
||||
buildServerMetadata: {
|
||||
...(existingBuildServerMetadata ?? {}),
|
||||
...newBuildServerMetadata,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
status: "INSTALLING",
|
||||
startedAt: new Date(),
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen((result) => {
|
||||
if (result.count === 0) {
|
||||
return errAsync({ type: "deployment_cannot_be_progressed" as const });
|
||||
}
|
||||
return okAsync({ id: deployment.id, status: "INSTALLING" as const });
|
||||
});
|
||||
};
|
||||
|
||||
const progressToBuilding = (
|
||||
deployment: Pick<WorkerDeployment, "id"> & { buildServerMetadata?: BuildServerMetadata }
|
||||
) => {
|
||||
const createRemoteBuildIfNeeded = deployment.buildServerMetadata?.isNativeBuild
|
||||
? okAsync(undefined)
|
||||
: fromPromise(createRemoteImageBuild(authenticatedEnv.project), (error) => ({
|
||||
type: "failed_to_create_remote_build" as const,
|
||||
cause: error,
|
||||
}));
|
||||
|
||||
const existingBuildServerMetadata = deployment.buildServerMetadata as
|
||||
| BuildServerMetadata
|
||||
| null
|
||||
| undefined;
|
||||
|
||||
return createRemoteBuildIfNeeded
|
||||
.andThen((externalBuildData) =>
|
||||
fromPromise(
|
||||
this._prisma.workerDeployment.updateMany({
|
||||
where: { id: deployment.id, status: "INSTALLING" }, // status could've changed in the meantime, we're not locking the row
|
||||
data: {
|
||||
...restUpdates,
|
||||
...(newBuildServerMetadata
|
||||
? {
|
||||
buildServerMetadata: {
|
||||
...(existingBuildServerMetadata ?? {}),
|
||||
...newBuildServerMetadata,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(externalBuildData ? { externalBuildData } : {}),
|
||||
status: "BUILDING",
|
||||
installedAt: new Date(),
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
)
|
||||
)
|
||||
.andThen((result) => {
|
||||
if (result.count === 0) {
|
||||
return errAsync({ type: "deployment_cannot_be_progressed" as const });
|
||||
}
|
||||
return okAsync({ id: deployment.id, status: "BUILDING" as const });
|
||||
});
|
||||
};
|
||||
|
||||
const extendTimeout = (deployment: Pick<WorkerDeployment, "id" | "status">) =>
|
||||
fromPromise(
|
||||
TimeoutDeploymentService.enqueue(
|
||||
deployment.id,
|
||||
deployment.status,
|
||||
deployment.status === "INSTALLING"
|
||||
? "Installing dependencies timed out"
|
||||
: "Building timed out",
|
||||
new Date(Date.now() + env.DEPLOY_TIMEOUT_MS)
|
||||
),
|
||||
(error) => ({
|
||||
type: "failed_to_extend_deployment_timeout" as const,
|
||||
cause: error,
|
||||
})
|
||||
);
|
||||
|
||||
return this.getDeployment(authenticatedEnv.projectId, friendlyId)
|
||||
.andThen(validateDeployment)
|
||||
.andThen((deployment) => {
|
||||
if (deployment.status === "PENDING") {
|
||||
return progressToInstalling(deployment);
|
||||
}
|
||||
return progressToBuilding(deployment);
|
||||
})
|
||||
.andThen(extendTimeout)
|
||||
.map(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels a deployment that is not yet in a final state.
|
||||
*
|
||||
* Only acts when the current status is not final. Not idempotent.
|
||||
*
|
||||
* @param authenticatedEnv The environment which the deployment belongs to.
|
||||
* @param friendlyId The friendly deployment ID.
|
||||
* @param data Cancelation reason.
|
||||
*/
|
||||
public cancelDeployment(
|
||||
authenticatedEnv: Pick<AuthenticatedEnvironment, "projectId">,
|
||||
friendlyId: string,
|
||||
data?: Partial<Pick<WorkerDeployment, "canceledReason">>
|
||||
) {
|
||||
const validateDeployment = (
|
||||
deployment: Pick<WorkerDeployment, "id" | "status" | "shortCode"> & {
|
||||
environment: { project: { externalRef: string } };
|
||||
}
|
||||
) => {
|
||||
if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
|
||||
logger.warn("Attempted cancelling deployment in a final state", {
|
||||
deployment,
|
||||
});
|
||||
return errAsync({ type: "deployment_cannot_be_cancelled" as const });
|
||||
}
|
||||
|
||||
return okAsync(deployment);
|
||||
};
|
||||
|
||||
const cancelDeployment = (
|
||||
deployment: Pick<WorkerDeployment, "id" | "shortCode"> & {
|
||||
environment: { project: { externalRef: string } };
|
||||
}
|
||||
) =>
|
||||
fromPromise(
|
||||
this._prisma.workerDeployment.updateMany({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
status: {
|
||||
notIn: FINAL_DEPLOYMENT_STATUSES, // status could've changed in the meantime, we're not locking the row
|
||||
},
|
||||
},
|
||||
data: {
|
||||
status: "CANCELED",
|
||||
canceledAt: new Date(),
|
||||
canceledReason: data?.canceledReason,
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
).andThen((result) => {
|
||||
if (result.count === 0) {
|
||||
return errAsync({ type: "deployment_cannot_be_cancelled" as const });
|
||||
}
|
||||
return okAsync({ deployment });
|
||||
});
|
||||
|
||||
const deleteTimeout = (deployment: Pick<WorkerDeployment, "id">) =>
|
||||
fromPromise(TimeoutDeploymentService.dequeue(deployment.id, this._prisma), (error) => ({
|
||||
type: "failed_to_delete_deployment_timeout" as const,
|
||||
cause: error,
|
||||
}));
|
||||
|
||||
return this.getDeployment(authenticatedEnv.projectId, friendlyId)
|
||||
.andThen(validateDeployment)
|
||||
.andThen(cancelDeployment)
|
||||
.andThen(({ deployment }) =>
|
||||
this.appendToEventLog(deployment.environment.project, deployment, [
|
||||
{
|
||||
type: "finalized",
|
||||
data: {
|
||||
result: "canceled",
|
||||
message: data?.canceledReason ?? undefined,
|
||||
},
|
||||
},
|
||||
])
|
||||
.orElse((error) => {
|
||||
logger.error("Failed to append event to deployment event log", { error });
|
||||
return okAsync(deployment);
|
||||
})
|
||||
.map(() => deployment)
|
||||
)
|
||||
.andThen(deleteTimeout)
|
||||
.map(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates registry credentials for a deployment. Returns an error if the deployment is in a final state.
|
||||
*
|
||||
* Uses the `platform` package, only available in cloud.
|
||||
*
|
||||
* @param authenticatedEnv The environment which the deployment belongs to.
|
||||
* @param friendlyId The friendly deployment ID.
|
||||
*/
|
||||
public generateRegistryCredentials(
|
||||
authenticatedEnv: Pick<AuthenticatedEnvironment, "projectId">,
|
||||
friendlyId: string
|
||||
) {
|
||||
const validateDeployment = (
|
||||
deployment: Pick<WorkerDeployment, "id" | "status" | "imageReference">
|
||||
) => {
|
||||
if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
|
||||
return errAsync({ type: "deployment_is_already_final" as const });
|
||||
}
|
||||
return okAsync(deployment);
|
||||
};
|
||||
|
||||
const getDeploymentRegion = (deployment: Pick<WorkerDeployment, "imageReference">) => {
|
||||
if (!deployment.imageReference) {
|
||||
return errAsync({ type: "deployment_has_no_image_reference" as const });
|
||||
}
|
||||
if (!deployment.imageReference.includes("amazonaws.com")) {
|
||||
return errAsync({ type: "registry_not_supported" as const });
|
||||
}
|
||||
|
||||
// we should connect the deployment to a region more explicitly in the future
|
||||
// for now we just use the image reference to determine the region
|
||||
if (deployment.imageReference.includes("us-east-1")) {
|
||||
return okAsync({ region: "us-east-1" as const });
|
||||
}
|
||||
if (deployment.imageReference.includes("eu-central-1")) {
|
||||
return okAsync({ region: "eu-central-1" as const });
|
||||
}
|
||||
|
||||
return errAsync({ type: "registry_region_not_supported" as const });
|
||||
};
|
||||
|
||||
const generateCredentials = ({ region }: { region: "us-east-1" | "eu-central-1" }) =>
|
||||
fromPromise(generateRegistryCredentials(authenticatedEnv.projectId, region), (error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})).andThen((result) => {
|
||||
if (!result || !result.success) {
|
||||
return errAsync({ type: "missing_registry_credentials" as const });
|
||||
}
|
||||
return okAsync({
|
||||
username: result.username,
|
||||
password: result.password,
|
||||
expiresAt: new Date(result.expiresAt),
|
||||
repositoryUri: result.repositoryUri,
|
||||
});
|
||||
});
|
||||
|
||||
return this.getDeployment(authenticatedEnv.projectId, friendlyId)
|
||||
.andThen(validateDeployment)
|
||||
.andThen(getDeploymentRegion)
|
||||
.andThen(generateCredentials);
|
||||
}
|
||||
|
||||
public enqueueBuild(
|
||||
authenticatedEnv: Pick<AuthenticatedEnvironment, "projectId">,
|
||||
deployment: Pick<WorkerDeployment, "friendlyId">,
|
||||
artifactKey: string,
|
||||
options: {
|
||||
skipPromotion?: boolean;
|
||||
configFilePath?: string;
|
||||
}
|
||||
) {
|
||||
return fromPromise(
|
||||
enqueueBuild(authenticatedEnv.projectId, deployment.friendlyId, artifactKey, options),
|
||||
(error) => ({
|
||||
type: "failed_to_enqueue_build" as const,
|
||||
cause: error,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
public appendToEventLog(
|
||||
project: Pick<Project, "externalRef">,
|
||||
deployment: Pick<WorkerDeployment, "shortCode">,
|
||||
events: DeploymentEvent[]
|
||||
): ResultAsync<
|
||||
undefined,
|
||||
{ type: "s2_is_disabled" } | { type: "failed_to_append_to_event_log"; cause: unknown }
|
||||
> {
|
||||
if (env.S2_ENABLED !== "1" || !s2) {
|
||||
return errAsync({ type: "s2_is_disabled" as const });
|
||||
}
|
||||
|
||||
const basin = s2.basin(env.S2_DEPLOYMENT_LOGS_BASIN_NAME);
|
||||
const stream = basin.stream(
|
||||
`projects/${project.externalRef}/deployments/${deployment.shortCode}`
|
||||
);
|
||||
|
||||
return fromPromise(
|
||||
stream.append(
|
||||
AppendInput.create(
|
||||
events.map((event) => AppendRecord.string({ body: JSON.stringify(event) }))
|
||||
)
|
||||
),
|
||||
(error) => ({
|
||||
type: "failed_to_append_to_event_log" as const,
|
||||
cause: error,
|
||||
})
|
||||
).map(() => undefined);
|
||||
}
|
||||
|
||||
public createEventStream(
|
||||
project: Pick<Project, "externalRef">,
|
||||
deployment: Pick<WorkerDeployment, "shortCode">
|
||||
): ResultAsync<
|
||||
{ basin: string; stream: string },
|
||||
{ type: "s2_is_disabled" } | { type: "failed_to_create_event_stream"; cause: unknown }
|
||||
> {
|
||||
if (env.S2_ENABLED !== "1" || !s2) {
|
||||
return errAsync({ type: "s2_is_disabled" as const });
|
||||
}
|
||||
const basin = s2.basin(env.S2_DEPLOYMENT_LOGS_BASIN_NAME);
|
||||
|
||||
return fromPromise(
|
||||
basin.streams.create({
|
||||
stream: `projects/${project.externalRef}/deployments/${deployment.shortCode}`,
|
||||
}),
|
||||
(error) => ({
|
||||
type: "failed_to_create_event_stream" as const,
|
||||
cause: error,
|
||||
})
|
||||
).map(() => ({
|
||||
basin: basin.name,
|
||||
stream: `projects/${project.externalRef}/deployments/${deployment.shortCode}`,
|
||||
}));
|
||||
}
|
||||
|
||||
public getEventStreamAccessToken(
|
||||
project: Pick<Project, "externalRef">
|
||||
): ResultAsync<string, { type: "s2_is_disabled" } | { type: "other"; cause: unknown }> {
|
||||
if (env.S2_ENABLED !== "1" || !s2) {
|
||||
return errAsync({ type: "s2_is_disabled" as const });
|
||||
}
|
||||
const basinName = env.S2_DEPLOYMENT_LOGS_BASIN_NAME;
|
||||
const redisKey = `${S2_TOKEN_KEY_PREFIX}${project.externalRef}`;
|
||||
|
||||
const getTokenFromCache = () =>
|
||||
fromPromise(s2TokenRedis.get(redisKey), (error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})).andThen((cachedToken) => {
|
||||
if (!cachedToken) {
|
||||
return errAsync({ type: "s2_token_cache_not_found" as const });
|
||||
}
|
||||
return okAsync(cachedToken);
|
||||
});
|
||||
|
||||
const issueS2Token = () =>
|
||||
fromPromise(
|
||||
s2.accessTokens.issue({
|
||||
id: `${project.externalRef}-${new Date().getTime()}`,
|
||||
expiresAt: new Date(Date.now() + 60 * 60 * 1000), // 1 hour
|
||||
scope: {
|
||||
ops: ["read"],
|
||||
basins: {
|
||||
exact: basinName,
|
||||
},
|
||||
streams: {
|
||||
prefix: `projects/${project.externalRef}/deployments/`,
|
||||
},
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
).map(({ accessToken }) => accessToken);
|
||||
|
||||
const cacheToken = (token: string) =>
|
||||
fromPromise(
|
||||
s2TokenRedis.setex(
|
||||
redisKey,
|
||||
59 * 60, // slightly shorter than the token validity period
|
||||
token
|
||||
),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
);
|
||||
|
||||
return getTokenFromCache().orElse(() =>
|
||||
issueS2Token().andThen((token) =>
|
||||
cacheToken(token)
|
||||
.map(() => token)
|
||||
.orElse((error) => {
|
||||
logger.error("Failed to cache S2 token", { error });
|
||||
return okAsync(token); // ignore the cache error
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private getDeployment(projectId: string, friendlyId: string) {
|
||||
return fromPromise(
|
||||
this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId,
|
||||
projectId,
|
||||
},
|
||||
select: {
|
||||
status: true,
|
||||
id: true,
|
||||
buildServerMetadata: true,
|
||||
imageReference: true,
|
||||
shortCode: true,
|
||||
environment: {
|
||||
include: {
|
||||
project: {
|
||||
select: {
|
||||
externalRef: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
(error) => ({
|
||||
type: "other" as const,
|
||||
cause: error,
|
||||
})
|
||||
)
|
||||
.andThen((deployment) => {
|
||||
if (!deployment) {
|
||||
return errAsync({ type: "deployment_not_found" as const });
|
||||
}
|
||||
return okAsync(deployment);
|
||||
})
|
||||
.map((deployment) => ({
|
||||
...deployment,
|
||||
buildServerMetadata: BuildServerMetadata.safeParse(deployment.buildServerMetadata).data,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { type WorkerDeploymentStatus } from "@trigger.dev/database";
|
||||
import { DeploymentService } from "./deployment.server";
|
||||
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
|
||||
|
||||
const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
|
||||
"CANCELED",
|
||||
"DEPLOYED",
|
||||
"FAILED",
|
||||
"TIMED_OUT",
|
||||
];
|
||||
|
||||
export class DeploymentIndexFailed extends BaseService {
|
||||
public async call(
|
||||
maybeFriendlyId: string,
|
||||
error: {
|
||||
name: string;
|
||||
message: string;
|
||||
stack?: string;
|
||||
stderr?: string;
|
||||
},
|
||||
overrideCompletion = false
|
||||
) {
|
||||
const isFriendlyId = maybeFriendlyId.startsWith("deployment_");
|
||||
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: isFriendlyId
|
||||
? {
|
||||
friendlyId: maybeFriendlyId,
|
||||
}
|
||||
: {
|
||||
id: maybeFriendlyId,
|
||||
},
|
||||
include: {
|
||||
environment: {
|
||||
include: {
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.error("Worker deployment not found", { maybeFriendlyId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
|
||||
if (overrideCompletion) {
|
||||
logger.error("No support for overriding final deployment statuses just yet", {
|
||||
id: deployment.id,
|
||||
status: deployment.status,
|
||||
previousError: deployment.errorData,
|
||||
incomingError: error,
|
||||
});
|
||||
}
|
||||
|
||||
logger.error("Worker deployment already in final state", {
|
||||
id: deployment.id,
|
||||
status: deployment.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const failedDeployment = await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
failedAt: new Date(),
|
||||
errorData: error,
|
||||
},
|
||||
});
|
||||
|
||||
recordDeploymentOutcome({
|
||||
status: "FAILED",
|
||||
deploymentFriendlyId: deployment.friendlyId,
|
||||
organizationId: deployment.environment.project.organizationId,
|
||||
projectId: deployment.environment.projectId,
|
||||
environmentId: deployment.environmentId,
|
||||
environmentType: deployment.environment.type,
|
||||
reason: error.message,
|
||||
});
|
||||
|
||||
const deploymentService = new DeploymentService();
|
||||
await deploymentService
|
||||
.appendToEventLog(deployment.environment.project, failedDeployment, [
|
||||
{
|
||||
type: "finalized",
|
||||
data: {
|
||||
result: "failed",
|
||||
message: error.message,
|
||||
},
|
||||
},
|
||||
])
|
||||
.orTee((error) => {
|
||||
logger.error("Failed to append failed deployment event to event log", { error });
|
||||
});
|
||||
|
||||
await PerformDeploymentAlertsService.enqueue(failedDeployment.id);
|
||||
|
||||
return failedDeployment;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { ServiceValidationError } from "./common.server";
|
||||
|
||||
type TaskIdResource = {
|
||||
id: string;
|
||||
filePath?: string;
|
||||
exportName?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the set of task ids that are defined more than once. All task types
|
||||
* (regular tasks, scheduled tasks, agents, etc.) share a single id namespace,
|
||||
* so a schedule and a regular task that use the same id count as a duplicate.
|
||||
*/
|
||||
export function findDuplicateTaskIds(tasks: Array<TaskIdResource>): string[] {
|
||||
const seen = new Set<string>();
|
||||
const duplicates = new Set<string>();
|
||||
|
||||
for (const task of tasks) {
|
||||
if (seen.has(task.id)) {
|
||||
duplicates.add(task.id);
|
||||
} else {
|
||||
seen.add(task.id);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(duplicates);
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws a customer-facing {@link ServiceValidationError} (HTTP 400) if any
|
||||
* task id is defined more than once, naming each offending id and the files it
|
||||
* was found in.
|
||||
*/
|
||||
export function assertNoDuplicateTaskIds(tasks: Array<TaskIdResource>): void {
|
||||
const duplicateTaskIds = findDuplicateTaskIds(tasks);
|
||||
|
||||
if (duplicateTaskIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const details = duplicateTaskIds
|
||||
.map((id) => {
|
||||
const locations = tasks
|
||||
.filter((task) => task.id === id)
|
||||
.map((task) => task.filePath ?? "unknown file")
|
||||
.join(", ");
|
||||
|
||||
return `"${id}" (defined in ${locations})`;
|
||||
})
|
||||
.join("; ");
|
||||
|
||||
throw new ServiceValidationError(
|
||||
`Duplicate task ids detected: ${details}. Each task must have a unique id across all task types (including scheduled tasks). Please rename one of them.`,
|
||||
400
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { enqueueRun } from "./enqueueRun.server";
|
||||
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { isV3Disabled } from "../engineDeprecation.server";
|
||||
|
||||
export class EnqueueDelayedRunService extends BaseService {
|
||||
public static async enqueue(runId: string, runAt?: Date) {
|
||||
await commonWorker.enqueue({
|
||||
job: "v3.enqueueDelayedRun",
|
||||
payload: { runId },
|
||||
availableAt: runAt,
|
||||
id: `v3.enqueueDelayed:${runId}`,
|
||||
});
|
||||
}
|
||||
|
||||
public static async reschedule(runId: string, runAt?: Date) {
|
||||
// We have to do this for now because it's possible that the workerQueue
|
||||
// was used when the run was first delayed, and EnqueueDelayedRunService.reschedule
|
||||
// is called from RescheduleTaskRunService, which allows the runAt to be changed
|
||||
// so if we don't dequeue the old job, we might end up with multiple jobs
|
||||
await workerQueue.dequeue(`v3.enqueueDelayedRun.${runId}`);
|
||||
|
||||
await commonWorker.enqueue({
|
||||
job: "v3.enqueueDelayedRun",
|
||||
payload: { runId },
|
||||
availableAt: runAt,
|
||||
id: `v3.enqueueDelayed:${runId}`,
|
||||
});
|
||||
}
|
||||
|
||||
public async call(runId: string) {
|
||||
const run = await this.runStore.findRun(
|
||||
{
|
||||
id: runId,
|
||||
},
|
||||
{
|
||||
include: {
|
||||
dependency: {
|
||||
include: {
|
||||
dependentBatchRun: {
|
||||
include: {
|
||||
dependentTaskAttempt: {
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dependentAttempt: {
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
logger.debug("Could not find delayed run to enqueue", {
|
||||
runId,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// v3 (engine V1) shutdown: don't enqueue delayed V1 runs into MarQS. v4 is unaffected.
|
||||
if (isV3Disabled() && run.engine === "V1") {
|
||||
logger.debug("[EnqueueDelayedRunService] Skipping enqueue for shut-down v3 run", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
const env = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId);
|
||||
|
||||
if (!env) {
|
||||
logger.debug("EnqueueDelayedRunService: environment not found", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status !== "DELAYED") {
|
||||
logger.debug("Delayed run cannot be enqueued because it's not in DELAYED status", {
|
||||
run,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: run.id,
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
queuedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
if (run.ttl) {
|
||||
const expireAt = parseNaturalLanguageDuration(run.ttl);
|
||||
|
||||
if (expireAt) {
|
||||
await ExpireEnqueuedRunService.enqueue(run.id, expireAt);
|
||||
}
|
||||
}
|
||||
|
||||
await enqueueRun({
|
||||
env,
|
||||
run: run,
|
||||
dependentRun:
|
||||
run.dependency?.dependentAttempt?.taskRun ??
|
||||
run.dependency?.dependentBatchRun?.dependentTaskAttempt?.taskRun,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { TaskRunError } from "@trigger.dev/core/v3/schemas";
|
||||
import { TaskRunErrorCodes } from "@trigger.dev/core/v3/schemas";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { marqs } from "../marqs/index.server";
|
||||
|
||||
export type EnqueueRunOptions = {
|
||||
env: AuthenticatedEnvironment;
|
||||
run: TaskRun;
|
||||
dependentRun?: { queue: string; id: string };
|
||||
};
|
||||
|
||||
export type EnqueueRunResult =
|
||||
| {
|
||||
ok: true;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: TaskRunError;
|
||||
};
|
||||
|
||||
export async function enqueueRun({
|
||||
env,
|
||||
run,
|
||||
dependentRun,
|
||||
}: EnqueueRunOptions): Promise<EnqueueRunResult> {
|
||||
// If this is a triggerAndWait or batchTriggerAndWait,
|
||||
// we need to add the parent run to the reserve concurrency set
|
||||
// to free up concurrency for the children to run
|
||||
// In the case of a recursive queue, reserving concurrency can fail, which means there is a deadlock and we need to fail the run
|
||||
|
||||
// TODO: reserveConcurrency can fail because of a deadlock, we need to handle that case
|
||||
const wasEnqueued = await marqs.enqueueMessage(
|
||||
env,
|
||||
run.queue,
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
projectId: env.projectId,
|
||||
environmentId: env.id,
|
||||
environmentType: env.type,
|
||||
},
|
||||
run.concurrencyKey ?? undefined,
|
||||
run.queueTimestamp ?? undefined,
|
||||
dependentRun
|
||||
? { messageId: dependentRun.id, recursiveQueue: dependentRun.queue === run.queue }
|
||||
: undefined
|
||||
);
|
||||
|
||||
if (!wasEnqueued) {
|
||||
const error = {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.RECURSIVE_WAIT_DEADLOCK,
|
||||
message: `This run will never execute because it was triggered recursively and the task has no remaining concurrency available`,
|
||||
} satisfies TaskRunError;
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { type PrismaClientOrTransaction, prisma } from "~/db.server";
|
||||
|
||||
type ErrorGroupIdentifier = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
taskIdentifier: string;
|
||||
errorFingerprint: string;
|
||||
};
|
||||
|
||||
export class ErrorGroupActions {
|
||||
constructor(private readonly _prisma: PrismaClientOrTransaction = prisma) {}
|
||||
|
||||
async resolveError(
|
||||
identifier: ErrorGroupIdentifier,
|
||||
params: {
|
||||
// Nullable: a resolve via an env API key has no acting user, so
|
||||
// `resolvedBy` stays null. The dashboard always passes a userId; the
|
||||
// API passes the `act.sub` user from a PAT/UAT-exchanged JWT, else null.
|
||||
userId?: string | null;
|
||||
resolvedInVersion?: string;
|
||||
}
|
||||
) {
|
||||
const where = {
|
||||
environmentId_taskIdentifier_errorFingerprint: {
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
},
|
||||
};
|
||||
|
||||
const now = new Date();
|
||||
|
||||
return this._prisma.errorGroupState.upsert({
|
||||
where,
|
||||
update: {
|
||||
status: "RESOLVED",
|
||||
resolvedAt: now,
|
||||
resolvedInVersion: params.resolvedInVersion ?? null,
|
||||
resolvedBy: params.userId ?? null,
|
||||
ignoredUntil: null,
|
||||
ignoredUntilOccurrenceRate: null,
|
||||
ignoredUntilTotalOccurrences: null,
|
||||
ignoredAtOccurrenceCount: null,
|
||||
ignoredAt: null,
|
||||
ignoredReason: null,
|
||||
ignoredByUserId: null,
|
||||
},
|
||||
create: {
|
||||
organizationId: identifier.organizationId,
|
||||
projectId: identifier.projectId,
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
status: "RESOLVED",
|
||||
resolvedAt: now,
|
||||
resolvedInVersion: params.resolvedInVersion ?? null,
|
||||
resolvedBy: params.userId ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async ignoreError(
|
||||
identifier: ErrorGroupIdentifier,
|
||||
params: {
|
||||
userId?: string | null;
|
||||
duration?: number;
|
||||
occurrenceRateThreshold?: number;
|
||||
totalOccurrencesThreshold?: number;
|
||||
occurrenceCountAtIgnoreTime?: number;
|
||||
reason?: string;
|
||||
}
|
||||
) {
|
||||
const where = {
|
||||
environmentId_taskIdentifier_errorFingerprint: {
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
},
|
||||
};
|
||||
|
||||
const now = new Date();
|
||||
const ignoredUntil = params.duration ? new Date(now.getTime() + params.duration) : null;
|
||||
|
||||
const data = {
|
||||
status: "IGNORED" as const,
|
||||
ignoredAt: now,
|
||||
ignoredUntil,
|
||||
ignoredUntilOccurrenceRate: params.occurrenceRateThreshold ?? null,
|
||||
ignoredUntilTotalOccurrences: params.totalOccurrencesThreshold ?? null,
|
||||
ignoredAtOccurrenceCount: params.occurrenceCountAtIgnoreTime ?? null,
|
||||
ignoredReason: params.reason ?? null,
|
||||
ignoredByUserId: params.userId ?? null,
|
||||
resolvedAt: null,
|
||||
resolvedInVersion: null,
|
||||
resolvedBy: null,
|
||||
};
|
||||
|
||||
return this._prisma.errorGroupState.upsert({
|
||||
where,
|
||||
update: data,
|
||||
create: {
|
||||
organizationId: identifier.organizationId,
|
||||
projectId: identifier.projectId,
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
...data,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async unresolveError(identifier: ErrorGroupIdentifier) {
|
||||
const where = {
|
||||
environmentId_taskIdentifier_errorFingerprint: {
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
},
|
||||
};
|
||||
|
||||
return this._prisma.errorGroupState.upsert({
|
||||
where,
|
||||
update: {
|
||||
status: "UNRESOLVED",
|
||||
resolvedAt: null,
|
||||
resolvedInVersion: null,
|
||||
resolvedBy: null,
|
||||
ignoredUntil: null,
|
||||
ignoredUntilOccurrenceRate: null,
|
||||
ignoredUntilTotalOccurrences: null,
|
||||
ignoredAtOccurrenceCount: null,
|
||||
ignoredAt: null,
|
||||
ignoredReason: null,
|
||||
ignoredByUserId: null,
|
||||
},
|
||||
create: {
|
||||
organizationId: identifier.organizationId,
|
||||
projectId: identifier.projectId,
|
||||
environmentId: identifier.environmentId,
|
||||
taskIdentifier: identifier.taskIdentifier,
|
||||
errorFingerprint: identifier.errorFingerprint,
|
||||
status: "UNRESOLVED",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { ownerEngine } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
export class ExecuteTasksWaitingForDeployService extends BaseService {
|
||||
public async call(backgroundWorkerId: string) {
|
||||
// Kill-switch for the legacy V1 WAITING_FOR_DEPLOY drain. Set to "1" to
|
||||
// neuter any jobs already enqueued (V2 has its own PENDING_VERSION path).
|
||||
if (env.LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_DISABLED === "1") {
|
||||
return;
|
||||
}
|
||||
|
||||
const backgroundWorker = await this._prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
id: backgroundWorkerId,
|
||||
},
|
||||
include: {
|
||||
runtimeEnvironment: {
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
},
|
||||
},
|
||||
tasks: {
|
||||
select: {
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!backgroundWorker) {
|
||||
logger.error("Background worker not found", { id: backgroundWorkerId });
|
||||
return;
|
||||
}
|
||||
|
||||
const maxCount = env.LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_SIZE;
|
||||
|
||||
const runsWaitingForDeploy = await this.runStore.findRuns(
|
||||
{
|
||||
where: {
|
||||
runtimeEnvironmentId: backgroundWorker.runtimeEnvironmentId,
|
||||
projectId: backgroundWorker.projectId,
|
||||
status: "WAITING_FOR_DEPLOY",
|
||||
taskIdentifier: {
|
||||
in: backgroundWorker.tasks.map((task) => task.slug),
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "asc",
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
taskIdentifier: true,
|
||||
concurrencyKey: true,
|
||||
queue: true,
|
||||
updatedAt: true,
|
||||
createdAt: true,
|
||||
},
|
||||
take: maxCount + 1,
|
||||
},
|
||||
this._replica
|
||||
);
|
||||
|
||||
if (!runsWaitingForDeploy.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Defense-in-depth: the open-predicate findRuns fan-out can select runs from
|
||||
// either DB, but the status flip below is a single control-plane updateMany. A
|
||||
// run-ops id (NEW-resident) run can only reach WAITING_FOR_DEPLOY via a misconfiguration
|
||||
// (it is a V1/cuid-only status — V2 uses PENDING_VERSION). Surface it loudly rather
|
||||
// than silently strand the run, and only mutate the LEGACY-resident runs the
|
||||
// control-plane client can actually reach.
|
||||
const newResidentRuns = runsWaitingForDeploy.filter((run) => ownerEngine(run.id) === "NEW");
|
||||
if (newResidentRuns.length) {
|
||||
logger.error(
|
||||
"WAITING_FOR_DEPLOY selected NEW-resident runs; skipping their control-plane status flip",
|
||||
{ runIds: newResidentRuns.map((run) => run.id) }
|
||||
);
|
||||
}
|
||||
const legacyRuns = runsWaitingForDeploy.filter((run) => !newResidentRuns.includes(run));
|
||||
|
||||
const pendingRuns = await this._prisma.taskRun.updateMany({
|
||||
where: {
|
||||
id: {
|
||||
in: legacyRuns.map((run) => run.id),
|
||||
},
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
if (pendingRuns.count) {
|
||||
logger.debug("Task runs waiting for deploy are now ready for execution", {
|
||||
tasks: legacyRuns.map((run) => run.id),
|
||||
total: pendingRuns.count,
|
||||
});
|
||||
}
|
||||
|
||||
// Only enqueue the runs whose status was actually flipped (the legacy set) — never
|
||||
// marqs-enqueue a NEW-resident run we couldn't transition out of WAITING_FOR_DEPLOY.
|
||||
for (const run of legacyRuns) {
|
||||
await marqs?.enqueueMessage(
|
||||
backgroundWorker.runtimeEnvironment,
|
||||
run.queue,
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
projectId: backgroundWorker.runtimeEnvironment.projectId,
|
||||
environmentId: backgroundWorker.runtimeEnvironment.id,
|
||||
environmentType: backgroundWorker.runtimeEnvironment.type,
|
||||
},
|
||||
run.concurrencyKey ?? undefined
|
||||
);
|
||||
}
|
||||
|
||||
if (runsWaitingForDeploy.length > maxCount) {
|
||||
await ExecuteTasksWaitingForDeployService.enqueue(
|
||||
backgroundWorkerId,
|
||||
new Date(Date.now() + env.LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_STAGGER_MS)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(backgroundWorkerId: string, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
id: `v3.executeTasksWaitingForDeploy:${backgroundWorkerId}`,
|
||||
job: "v3.executeTasksWaitingForDeploy",
|
||||
payload: {
|
||||
backgroundWorkerId,
|
||||
},
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import { getEventRepositoryForStore } from "../eventRepository/index.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { isV3Disabled } from "../engineDeprecation.server";
|
||||
|
||||
export class ExpireEnqueuedRunService extends BaseService {
|
||||
public static async ack(runId: string, tx?: PrismaClientOrTransaction) {
|
||||
// We don't "dequeue" from the workerQueue here because it would be redundant and if this service
|
||||
// is called for a run that has already started, nothing happens
|
||||
await commonWorker.ack(`v3.expireRun:${runId}`);
|
||||
}
|
||||
|
||||
public static async enqueue(runId: string, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
job: "v3.expireRun",
|
||||
payload: { runId },
|
||||
availableAt: runAt,
|
||||
id: `v3.expireRun:${runId}`,
|
||||
});
|
||||
}
|
||||
|
||||
public async call(runId: string) {
|
||||
const run = await this.runStore.findRun(
|
||||
{
|
||||
id: runId,
|
||||
},
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
engine: true,
|
||||
lockedAt: true,
|
||||
ttl: true,
|
||||
taskEventStore: true,
|
||||
runtimeEnvironmentId: true,
|
||||
friendlyId: true,
|
||||
traceId: true,
|
||||
spanId: true,
|
||||
parentSpanId: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
taskIdentifier: true,
|
||||
projectId: true,
|
||||
organizationId: true,
|
||||
isTest: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
logger.debug("Could not find enqueued run to expire", {
|
||||
runId,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// v3 (engine V1) shutdown: skip expiring abandoned V1 runs. v4 is unaffected.
|
||||
if (isV3Disabled() && run.engine === "V1") {
|
||||
logger.debug("[ExpireEnqueuedRunService] Skipping expiry for shut-down v3 run", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
const env = await controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId);
|
||||
|
||||
if (!env) {
|
||||
logger.debug("ExpireEnqueuedRunService: environment not found", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status !== "PENDING") {
|
||||
logger.debug("Run cannot be expired because it's not in PENDING status", {
|
||||
run,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.lockedAt) {
|
||||
logger.debug("Run cannot be expired because it's locked", {
|
||||
run,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("Expiring enqueued run", {
|
||||
run,
|
||||
});
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: run.id,
|
||||
status: "EXPIRED",
|
||||
expiredAt: new Date(),
|
||||
completedAt: new Date(),
|
||||
attemptStatus: "FAILED",
|
||||
error: {
|
||||
type: "STRING_ERROR",
|
||||
raw: `Run expired because the TTL (${run.ttl}) was reached`,
|
||||
},
|
||||
});
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
run.taskEventStore,
|
||||
env.organizationId
|
||||
);
|
||||
|
||||
if (run.ttl) {
|
||||
const [completeExpiredRunEventError] = await tryCatch(
|
||||
eventRepository.completeExpiredRunEvent({
|
||||
run,
|
||||
endTime: new Date(),
|
||||
ttl: run.ttl,
|
||||
})
|
||||
);
|
||||
|
||||
if (completeExpiredRunEventError) {
|
||||
logger.error("[ExpireEnqueuedRunService] Failed to complete expired run event", {
|
||||
error: completeExpiredRunEventError,
|
||||
runId: run.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { type WorkerDeploymentStatus } from "@trigger.dev/database";
|
||||
import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { DeploymentService } from "./deployment.server";
|
||||
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
|
||||
|
||||
export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
|
||||
"CANCELED",
|
||||
"DEPLOYED",
|
||||
"FAILED",
|
||||
"TIMED_OUT",
|
||||
];
|
||||
|
||||
export class FailDeploymentService extends BaseService {
|
||||
public async call(
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
friendlyId: string,
|
||||
params: FailDeploymentRequestBody
|
||||
) {
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId,
|
||||
environmentId: authenticatedEnv.id,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.error("Worker deployment not found", { friendlyId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
|
||||
logger.error("Worker deployment already in final state", {
|
||||
id: deployment.id,
|
||||
friendlyId,
|
||||
status: deployment.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const failedDeployment = await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
failedAt: new Date(),
|
||||
errorData: params.error,
|
||||
},
|
||||
});
|
||||
|
||||
recordDeploymentOutcome({
|
||||
status: "FAILED",
|
||||
deploymentFriendlyId: friendlyId,
|
||||
organizationId: authenticatedEnv.organizationId,
|
||||
projectId: authenticatedEnv.projectId,
|
||||
environmentId: authenticatedEnv.id,
|
||||
environmentType: authenticatedEnv.type,
|
||||
reason: params.error.message,
|
||||
});
|
||||
|
||||
const deploymentService = new DeploymentService();
|
||||
await deploymentService
|
||||
.appendToEventLog(authenticatedEnv.project, failedDeployment, [
|
||||
{
|
||||
type: "finalized",
|
||||
data: {
|
||||
result: "failed",
|
||||
message: params.error.message,
|
||||
},
|
||||
},
|
||||
])
|
||||
.orTee((error) => {
|
||||
logger.error("Failed to append failed deployment event to event log", { error });
|
||||
});
|
||||
|
||||
await PerformDeploymentAlertsService.enqueue(failedDeployment.id);
|
||||
|
||||
return failedDeployment;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
import type { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
||||
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { ChangeCurrentDeploymentService } from "./changeCurrentDeployment.server";
|
||||
import { projectPubSub } from "./projectPubSub.server";
|
||||
import { FailDeploymentService } from "./failDeployment.server";
|
||||
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
||||
import { DeploymentService } from "./deployment.server";
|
||||
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
|
||||
import { engine } from "../runEngine.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
|
||||
export class FinalizeDeploymentService extends BaseService {
|
||||
public async call(
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
id: string,
|
||||
body: FinalizeDeploymentRequestBody
|
||||
) {
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId: id,
|
||||
environmentId: authenticatedEnv.id,
|
||||
},
|
||||
include: {
|
||||
worker: {
|
||||
include: {
|
||||
tasks: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.error("Worker deployment not found", { id });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!deployment.worker) {
|
||||
logger.error("Worker deployment does not have a worker", { id });
|
||||
|
||||
const failService = new FailDeploymentService();
|
||||
await failService.call(authenticatedEnv, deployment.friendlyId, {
|
||||
error: {
|
||||
name: "MissingWorker",
|
||||
message: "Deployment does not have a worker",
|
||||
},
|
||||
});
|
||||
|
||||
throw new ServiceValidationError("Worker deployment does not have a worker");
|
||||
}
|
||||
|
||||
if (deployment.status === "DEPLOYED") {
|
||||
logger.debug("Worker deployment is already deployed", { id });
|
||||
|
||||
return deployment;
|
||||
}
|
||||
|
||||
if (deployment.status !== "DEPLOYING") {
|
||||
logger.error("Worker deployment is not in DEPLOYING status", { id });
|
||||
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
|
||||
}
|
||||
|
||||
const imageDigest = validatedImageDigest(body.imageDigest);
|
||||
|
||||
// Link the deployment with the background worker
|
||||
const finalizedDeployment = await this._prisma.workerDeployment.update({
|
||||
where: {
|
||||
id: deployment.id,
|
||||
},
|
||||
data: {
|
||||
status: "DEPLOYED",
|
||||
deployedAt: new Date(),
|
||||
// Only add the digest, if any
|
||||
imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
recordDeploymentOutcome({
|
||||
status: "DEPLOYED",
|
||||
deploymentFriendlyId: deployment.friendlyId,
|
||||
organizationId: authenticatedEnv.organizationId,
|
||||
projectId: authenticatedEnv.projectId,
|
||||
environmentId: authenticatedEnv.id,
|
||||
environmentType: authenticatedEnv.type,
|
||||
});
|
||||
|
||||
const deploymentService = new DeploymentService();
|
||||
await deploymentService
|
||||
.appendToEventLog(authenticatedEnv.project, finalizedDeployment, [
|
||||
{
|
||||
type: "finalized",
|
||||
data: {
|
||||
result: "succeeded",
|
||||
},
|
||||
},
|
||||
])
|
||||
.orTee((error) => {
|
||||
logger.error("Failed to append finalized deployment event to event log", { error });
|
||||
});
|
||||
|
||||
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
|
||||
|
||||
if (typeof body.skipPromotion === "undefined" || !body.skipPromotion) {
|
||||
const promotionService = new ChangeCurrentDeploymentService();
|
||||
|
||||
await promotionService.call(finalizedDeployment, "promote");
|
||||
}
|
||||
|
||||
try {
|
||||
//send a notification that a new worker has been created
|
||||
await projectPubSub.publish(
|
||||
`project:${authenticatedEnv.projectId}:env:${authenticatedEnv.id}`,
|
||||
"WORKER_CREATED",
|
||||
{
|
||||
environmentId: authenticatedEnv.id,
|
||||
environmentType: authenticatedEnv.type,
|
||||
createdAt: authenticatedEnv.createdAt,
|
||||
taskCount: deployment.worker.tasks.length,
|
||||
type: "deployed",
|
||||
}
|
||||
);
|
||||
|
||||
await updateEnvConcurrencyLimits(authenticatedEnv);
|
||||
} catch (err) {
|
||||
logger.error("Failed to publish WORKER_CREATED event", { err });
|
||||
}
|
||||
|
||||
if (finalizedDeployment.imageReference) {
|
||||
socketIo.providerNamespace.emit("PRE_PULL_DEPLOYMENT", {
|
||||
version: "v1",
|
||||
imageRef: finalizedDeployment.imageReference,
|
||||
shortCode: finalizedDeployment.shortCode,
|
||||
// identifiers
|
||||
deploymentId: finalizedDeployment.id,
|
||||
envId: authenticatedEnv.id,
|
||||
envType: authenticatedEnv.type,
|
||||
orgId: authenticatedEnv.organizationId,
|
||||
projectId: finalizedDeployment.projectId,
|
||||
});
|
||||
}
|
||||
|
||||
if (deployment.worker.engine === "V2") {
|
||||
const [schedulePendingVersionsError] = await tryCatch(
|
||||
engine.scheduleEnqueueRunsForBackgroundWorker(deployment.worker.id)
|
||||
);
|
||||
|
||||
if (schedulePendingVersionsError) {
|
||||
logger.error("Error scheduling pending versions", {
|
||||
error: schedulePendingVersionsError,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await PerformDeploymentAlertsService.enqueue(deployment.id);
|
||||
|
||||
return finalizedDeployment;
|
||||
}
|
||||
}
|
||||
|
||||
function validatedImageDigest(imageDigest?: string): string | undefined {
|
||||
if (!imageDigest) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(imageDigest.trim())) {
|
||||
logger.error("Invalid image digest", { imageDigest });
|
||||
return;
|
||||
}
|
||||
|
||||
return imageDigest.trim();
|
||||
}
|
||||
@@ -0,0 +1,397 @@
|
||||
import {
|
||||
ExternalBuildData,
|
||||
type FinalizeDeploymentRequestBody,
|
||||
} from "@trigger.dev/core/v3/schemas";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { env } from "~/env.server";
|
||||
import { depot as execDepot } from "@depot/cli";
|
||||
import { FinalizeDeploymentService } from "./finalizeDeployment.server";
|
||||
import { remoteBuildsEnabled } from "../remoteImageBuilder.server";
|
||||
import { getEcrAuthToken, isEcrRegistry } from "../getDeploymentImageRef.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { getRegistryConfig, type RegistryConfig } from "../registryConfig.server";
|
||||
import { ComputeTemplateCreationService } from "./computeTemplateCreation.server";
|
||||
import { ecrImageExists } from "./verifyDeploymentImage.server";
|
||||
|
||||
export class FinalizeDeploymentV2Service extends BaseService {
|
||||
public async call(
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
id: string,
|
||||
body: FinalizeDeploymentRequestBody,
|
||||
writer?: WritableStreamDefaultWriter
|
||||
) {
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
friendlyId: id,
|
||||
environmentId: authenticatedEnv.id,
|
||||
},
|
||||
select: {
|
||||
status: true,
|
||||
id: true,
|
||||
version: true,
|
||||
externalBuildData: true,
|
||||
environment: true,
|
||||
imageReference: true,
|
||||
type: true,
|
||||
worker: {
|
||||
select: {
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
logger.error("Worker deployment not found", { id });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!deployment.worker) {
|
||||
logger.error("Worker deployment does not have a worker", { id });
|
||||
throw new ServiceValidationError("Worker deployment does not have a worker");
|
||||
}
|
||||
|
||||
if (deployment.status === "DEPLOYED") {
|
||||
logger.debug("Worker deployment is already deployed", { id });
|
||||
return deployment;
|
||||
}
|
||||
|
||||
if (deployment.status !== "DEPLOYING") {
|
||||
logger.error("Worker deployment is not in DEPLOYING status", { id });
|
||||
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeDeploymentService();
|
||||
|
||||
// If remote builds are not enabled, skip image push and go straight to template + finalize
|
||||
if (!remoteBuildsEnabled() || body.skipPushToRegistry) {
|
||||
if (body.skipPushToRegistry) {
|
||||
logger.debug("Skipping push to registry during deployment finalization", {
|
||||
deployment,
|
||||
});
|
||||
}
|
||||
|
||||
// The CLI claims the image is already in the registry (local build, or a
|
||||
// self-hosted setup). Verify before promoting so we never mark a
|
||||
// deployment DEPLOYED when nothing was actually pushed.
|
||||
await this.#assertImagePullable(deployment, body);
|
||||
|
||||
await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer);
|
||||
return finalizeService.call(authenticatedEnv, id, body);
|
||||
}
|
||||
|
||||
const externalBuildData = deployment.externalBuildData
|
||||
? ExternalBuildData.safeParse(deployment.externalBuildData)
|
||||
: undefined;
|
||||
|
||||
if (!externalBuildData) {
|
||||
throw new ServiceValidationError("External build data is missing");
|
||||
}
|
||||
|
||||
if (!externalBuildData.success) {
|
||||
throw new ServiceValidationError("External build data is invalid");
|
||||
}
|
||||
|
||||
const isV4Deployment = deployment.type === "MANAGED";
|
||||
const registryConfig = getRegistryConfig(isV4Deployment);
|
||||
|
||||
// For non-ECR registries, username and password are required upfront
|
||||
if (
|
||||
!isEcrRegistry(registryConfig.host) &&
|
||||
(!registryConfig.username || !registryConfig.password)
|
||||
) {
|
||||
throw new ServiceValidationError("Missing deployment registry credentials");
|
||||
}
|
||||
|
||||
if (!env.DEPOT_TOKEN) {
|
||||
throw new ServiceValidationError("Missing depot token");
|
||||
}
|
||||
|
||||
// All new deployments will set the image reference at creation time
|
||||
if (!deployment.imageReference) {
|
||||
throw new ServiceValidationError("Missing image reference");
|
||||
}
|
||||
|
||||
logger.debug("Pushing image to registry", { id, deployment, body });
|
||||
|
||||
const pushResult = await executePushToRegistry(
|
||||
{
|
||||
depot: {
|
||||
buildId: externalBuildData.data.buildId,
|
||||
orgToken: env.DEPOT_TOKEN,
|
||||
projectId: externalBuildData.data.projectId,
|
||||
},
|
||||
registry: registryConfig,
|
||||
deployment: {
|
||||
version: deployment.version,
|
||||
environmentSlug: deployment.environment.slug,
|
||||
projectExternalRef: deployment.worker.project.externalRef,
|
||||
imageReference: deployment.imageReference,
|
||||
},
|
||||
},
|
||||
writer
|
||||
);
|
||||
|
||||
if (!pushResult.ok) {
|
||||
throw new ServiceValidationError(pushResult.error);
|
||||
}
|
||||
|
||||
logger.debug("Image pushed to registry", {
|
||||
id,
|
||||
deployment,
|
||||
body,
|
||||
pushedImage: pushResult.image,
|
||||
});
|
||||
|
||||
// Belt and suspenders: confirm the push actually landed before promoting.
|
||||
await this.#assertImagePullable(deployment, body);
|
||||
|
||||
await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer);
|
||||
return finalizeService.call(authenticatedEnv, id, body);
|
||||
}
|
||||
|
||||
async #assertImagePullable(
|
||||
deployment: { imageReference: string | null; type: string | null },
|
||||
body: FinalizeDeploymentRequestBody
|
||||
): Promise<void> {
|
||||
if (!env.DEPLOY_IMAGE_VERIFICATION_ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!deployment.imageReference) {
|
||||
return;
|
||||
}
|
||||
|
||||
const registryConfig = getRegistryConfig(deployment.type === "MANAGED");
|
||||
|
||||
// ECR-only: non-ECR (self-hosted) registries can't be checked this way, so skip.
|
||||
if (!isEcrRegistry(registryConfig.host)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await ecrImageExists({
|
||||
imageReference: deployment.imageReference,
|
||||
imageDigest: body.imageDigest,
|
||||
registryConfig,
|
||||
});
|
||||
|
||||
if (result === "missing") {
|
||||
throw new ServiceValidationError(
|
||||
"Deployment image was not found in the registry. It may not have been pushed (for example a local build without a push, or a push to a different registry). Aborting the deploy to avoid promoting a version that cannot start."
|
||||
);
|
||||
}
|
||||
|
||||
if (result === "nonconformant") {
|
||||
throw new ServiceValidationError(
|
||||
"Deployment image is not runnable: it contains zstd-compressed layers inside a Docker (v2s2) manifest, which the container runtime cannot pull. This typically comes from an outdated CLI version. Please upgrade to the latest trigger.dev CLI and re-deploy."
|
||||
);
|
||||
}
|
||||
|
||||
// Fail closed: if we can't confirm the image is present, don't promote a version
|
||||
// that might not start. Set DEPLOY_IMAGE_VERIFICATION_ENABLED=0 for out-of-band pushes.
|
||||
if (result === "unknown") {
|
||||
throw new ServiceValidationError(
|
||||
"Could not verify the deployment image exists in the registry. Aborting the deploy."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #createTemplateIfNeeded(
|
||||
deployment: { imageReference: string | null; worker: { project: { id: string } } | null },
|
||||
deploymentFriendlyId: string,
|
||||
authenticatedEnv: AuthenticatedEnvironment,
|
||||
writer?: WritableStreamDefaultWriter
|
||||
): Promise<void> {
|
||||
if (!deployment.imageReference || !deployment.worker) {
|
||||
return;
|
||||
}
|
||||
|
||||
const templateService = new ComputeTemplateCreationService();
|
||||
await templateService.handleDeployTemplate({
|
||||
projectId: deployment.worker.project.id,
|
||||
imageReference: deployment.imageReference,
|
||||
deploymentFriendlyId,
|
||||
authenticatedEnv,
|
||||
prisma: this._prisma,
|
||||
writer,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
type ExecutePushToRegistryOptions = {
|
||||
depot: {
|
||||
buildId: string;
|
||||
orgToken: string;
|
||||
projectId: string;
|
||||
};
|
||||
registry: RegistryConfig;
|
||||
deployment: {
|
||||
version: string;
|
||||
environmentSlug: string;
|
||||
projectExternalRef: string;
|
||||
imageReference: string;
|
||||
};
|
||||
};
|
||||
|
||||
type ExecutePushResult =
|
||||
| {
|
||||
ok: true;
|
||||
image: string;
|
||||
logs: string;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: string;
|
||||
logs: string;
|
||||
};
|
||||
|
||||
async function executePushToRegistry(
|
||||
{ depot, registry, deployment }: ExecutePushToRegistryOptions,
|
||||
writer?: WritableStreamDefaultWriter
|
||||
): Promise<ExecutePushResult> {
|
||||
// Step 1: We need to "login" to the registry
|
||||
const [loginError, configDir] = await tryCatch(ensureLoggedIntoDockerRegistry(registry));
|
||||
|
||||
if (loginError) {
|
||||
logger.error("Failed to login to registry", {
|
||||
deployment,
|
||||
registryHost: registry.host,
|
||||
error: loginError.message,
|
||||
});
|
||||
|
||||
return {
|
||||
ok: false as const,
|
||||
error: "Failed to login to registry",
|
||||
logs: "",
|
||||
};
|
||||
}
|
||||
|
||||
const imageTag = deployment.imageReference;
|
||||
|
||||
// Step 2: We need to run the depot push command
|
||||
const childProcess = execDepot(["push", depot.buildId, "-t", imageTag, "--progress", "plain"], {
|
||||
env: {
|
||||
NODE_ENV: process.env.NODE_ENV,
|
||||
DEPOT_TOKEN: depot.orgToken,
|
||||
DEPOT_PROJECT_ID: depot.projectId,
|
||||
DEPOT_NO_SUMMARY_LINK: "1",
|
||||
DEPOT_NO_UPDATE_NOTIFIER: "1",
|
||||
DOCKER_CONFIG: configDir,
|
||||
},
|
||||
});
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
try {
|
||||
const processCode = await new Promise<number | null>((res, rej) => {
|
||||
// For some reason everything is output on stderr, not stdout
|
||||
childProcess.stderr?.on("data", async (data: Buffer) => {
|
||||
const text = data.toString();
|
||||
|
||||
// Emitted data chunks can contain multiple lines. Remove empty lines.
|
||||
const lines = text.split("\n").filter(Boolean);
|
||||
|
||||
errors.push(...lines);
|
||||
logger.debug(text, { deployment });
|
||||
|
||||
// Now we can write strings directly
|
||||
if (writer) {
|
||||
for (const line of lines) {
|
||||
await writer.write(`event: log\ndata: ${JSON.stringify({ message: line })}\n\n`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
childProcess.on("error", (e) => rej(e));
|
||||
childProcess.on("close", (code) => res(code));
|
||||
});
|
||||
|
||||
const logs = extractLogs(errors);
|
||||
|
||||
if (processCode !== 0) {
|
||||
return {
|
||||
ok: false as const,
|
||||
error: `Error pushing image`,
|
||||
logs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
image: imageTag,
|
||||
logs,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false as const,
|
||||
error: e instanceof Error ? e.message : JSON.stringify(e),
|
||||
logs: extractLogs(errors),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureLoggedIntoDockerRegistry(registryConfig: RegistryConfig) {
|
||||
const tmpDir = await createTempDir();
|
||||
const dockerConfigPath = join(tmpDir, "config.json");
|
||||
|
||||
let auth: { username: string; password: string };
|
||||
|
||||
// If this is an ECR registry, get fresh credentials
|
||||
if (isEcrRegistry(registryConfig.host)) {
|
||||
auth = await getEcrAuthToken({
|
||||
registryHost: registryConfig.host,
|
||||
assumeRole: registryConfig.ecrAssumeRoleArn
|
||||
? {
|
||||
roleArn: registryConfig.ecrAssumeRoleArn,
|
||||
externalId: registryConfig.ecrAssumeRoleExternalId,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
} else if (!registryConfig.username || !registryConfig.password) {
|
||||
throw new Error("Authentication required for non-ECR registry");
|
||||
} else {
|
||||
auth = {
|
||||
username: registryConfig.username,
|
||||
password: registryConfig.password,
|
||||
};
|
||||
}
|
||||
|
||||
await writeJSONFile(dockerConfigPath, {
|
||||
auths: {
|
||||
[registryConfig.host]: {
|
||||
auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug(`Writing docker config to ${dockerConfigPath}`);
|
||||
|
||||
return tmpDir;
|
||||
}
|
||||
|
||||
// Create a temporary directory within the OS's temp directory
|
||||
async function createTempDir(): Promise<string> {
|
||||
// Generate a unique temp directory path
|
||||
const tempDirPath: string = join(tmpdir(), "trigger-");
|
||||
|
||||
// Create the temp directory synchronously and return the path
|
||||
const directory = await mkdtemp(tempDirPath);
|
||||
|
||||
return directory;
|
||||
}
|
||||
|
||||
async function writeJSONFile(path: string, json: any, pretty = false) {
|
||||
await writeFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined), "utf8");
|
||||
}
|
||||
|
||||
function extractLogs(outputs: string[]) {
|
||||
// Remove empty lines
|
||||
const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== "");
|
||||
|
||||
return cleanedOutputs.map((line) => line.trim()).join("\n");
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
import { type FlushedRunMetadata, type TaskRunError, sanitizeError } from "@trigger.dev/core/v3";
|
||||
import { type Prisma, type TaskRun } from "@trigger.dev/database";
|
||||
import { findQueueInEnvironment } from "~/models/taskQueue.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { updateMetadataService } from "~/services/metadata/updateMetadataInstance.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import {
|
||||
type FINAL_ATTEMPT_STATUSES,
|
||||
isFailedRunStatus,
|
||||
isFatalRunStatus,
|
||||
type FINAL_RUN_STATUSES,
|
||||
} from "../taskStatus";
|
||||
import { PerformTaskRunAlertsService } from "./alerts/performTaskRunAlerts.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { completeBatchTaskRunItemV3 } from "./batchTriggerV3.server";
|
||||
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
|
||||
import { ResumeBatchRunService } from "./resumeBatchRun.server";
|
||||
import { ResumeDependentParentsService } from "./resumeDependentParents.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
|
||||
type BaseInput = {
|
||||
id: string;
|
||||
status?: FINAL_RUN_STATUSES;
|
||||
expiredAt?: Date;
|
||||
completedAt?: Date;
|
||||
attemptStatus?: FINAL_ATTEMPT_STATUSES;
|
||||
error?: TaskRunError;
|
||||
metadata?: FlushedRunMetadata;
|
||||
env?: AuthenticatedEnvironment;
|
||||
bulkActionId?: string;
|
||||
};
|
||||
|
||||
type InputWithInclude<T extends Prisma.TaskRunInclude> = BaseInput & {
|
||||
include: T;
|
||||
};
|
||||
|
||||
type InputWithoutInclude = BaseInput & {
|
||||
include?: undefined;
|
||||
};
|
||||
|
||||
type Output<T extends Prisma.TaskRunInclude | undefined> = T extends Prisma.TaskRunInclude
|
||||
? Prisma.TaskRunGetPayload<{ include: T }>
|
||||
: TaskRun;
|
||||
|
||||
export class FinalizeTaskRunService extends BaseService {
|
||||
public async call<T extends Prisma.TaskRunInclude | undefined>({
|
||||
id,
|
||||
status,
|
||||
expiredAt,
|
||||
completedAt,
|
||||
bulkActionId,
|
||||
include,
|
||||
attemptStatus,
|
||||
error,
|
||||
metadata,
|
||||
env,
|
||||
}: T extends Prisma.TaskRunInclude ? InputWithInclude<T> : InputWithoutInclude): Promise<
|
||||
Output<T>
|
||||
> {
|
||||
logger.debug("Finalizing run marqs ack", {
|
||||
id,
|
||||
status,
|
||||
expiredAt,
|
||||
completedAt,
|
||||
});
|
||||
await marqs?.acknowledgeMessage(id, "FinalTaskRunService call");
|
||||
|
||||
logger.debug("Finalizing run updating run status", {
|
||||
id,
|
||||
status,
|
||||
expiredAt,
|
||||
completedAt,
|
||||
});
|
||||
|
||||
if (metadata) {
|
||||
try {
|
||||
await updateMetadataService.call(id, metadata, env);
|
||||
} catch (e) {
|
||||
logger.error("[FinalizeTaskRunService] Failed to update metadata", {
|
||||
taskRun: id,
|
||||
error:
|
||||
e instanceof Error
|
||||
? {
|
||||
name: e.name,
|
||||
message: e.message,
|
||||
stack: e.stack,
|
||||
}
|
||||
: e,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Error is written in the same update as the status: a separate later write races realtime,
|
||||
// which shuts the stream down on the final status before the error lands, losing it.
|
||||
const taskRunError = error ? sanitizeError(error) : undefined;
|
||||
|
||||
const run = await this._prisma.taskRun.update({
|
||||
where: { id },
|
||||
data: {
|
||||
status,
|
||||
expiredAt,
|
||||
completedAt,
|
||||
error: taskRunError,
|
||||
bulkActionGroupIds: bulkActionId
|
||||
? {
|
||||
push: bulkActionId,
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
...(include ? { include } : {}),
|
||||
});
|
||||
|
||||
if (run.ttl) {
|
||||
await ExpireEnqueuedRunService.ack(run.id);
|
||||
}
|
||||
|
||||
if (attemptStatus || error) {
|
||||
await this.finalizeAttempt({ attemptStatus, error, run });
|
||||
}
|
||||
|
||||
try {
|
||||
await this.#finalizeBatch(run);
|
||||
} catch (finalizeBatchError) {
|
||||
logger.error("FinalizeTaskRunService: Failed to finalize batch", {
|
||||
runId: run.id,
|
||||
error: finalizeBatchError,
|
||||
});
|
||||
}
|
||||
|
||||
const resumeService = new ResumeDependentParentsService(this._prisma);
|
||||
const result = await resumeService.call({ id: run.id });
|
||||
|
||||
if (result.success) {
|
||||
logger.log("FinalizeTaskRunService: Resumed dependent parents", { result, run: run.id });
|
||||
} else {
|
||||
logger.error("FinalizeTaskRunService: Failed to resume dependent parents", {
|
||||
result,
|
||||
run: run.id,
|
||||
});
|
||||
}
|
||||
|
||||
if (isFailedRunStatus(run.status)) {
|
||||
await PerformTaskRunAlertsService.enqueue(run.id);
|
||||
}
|
||||
|
||||
if (isFatalRunStatus(run.status)) {
|
||||
logger.warn("FinalizeTaskRunService: Fatal status", { runId: run.id, status: run.status });
|
||||
|
||||
const extendedRun = await this.runStore.findRun(
|
||||
{ id: run.id },
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
runtimeEnvironmentId: true,
|
||||
lockedToVersionId: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
const extendedEnv = extendedRun
|
||||
? await controlPlaneResolver.resolveEnv(extendedRun.runtimeEnvironmentId)
|
||||
: null;
|
||||
const extendedLockedWorker = extendedRun
|
||||
? await controlPlaneResolver.resolveRunLockedWorker({
|
||||
lockedToVersionId: extendedRun.lockedToVersionId,
|
||||
})
|
||||
: null;
|
||||
|
||||
if (extendedRun && extendedEnv && extendedEnv.type !== "DEVELOPMENT") {
|
||||
logger.warn("FinalizeTaskRunService: Fatal status, requesting worker exit", {
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
// Signal to exit any leftover containers
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
|
||||
version: "v1",
|
||||
runId: run.id,
|
||||
// Give the run a few seconds to exit to complete any flushing etc
|
||||
delayInMs: extendedLockedWorker?.lockedToVersion?.supportsLazyAttempts
|
||||
? 5_000
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return run as Output<T>;
|
||||
}
|
||||
|
||||
async #finalizeBatch(run: TaskRun) {
|
||||
if (!run.batchId) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug("FinalizeTaskRunService: Finalizing batch", { runId: run.id });
|
||||
|
||||
const environment = await this._prisma.runtimeEnvironment.findFirst({
|
||||
where: {
|
||||
id: run.runtimeEnvironmentId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!environment) {
|
||||
return;
|
||||
}
|
||||
|
||||
const batchItems = await this._prisma.batchTaskRunItem.findMany({
|
||||
where: {
|
||||
taskRunId: run.id,
|
||||
},
|
||||
include: {
|
||||
batchTaskRun: {
|
||||
select: {
|
||||
id: true,
|
||||
dependentTaskAttemptId: true,
|
||||
batchVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (batchItems.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (batchItems.length > 10) {
|
||||
logger.error("FinalizeTaskRunService: More than 10 batch items", {
|
||||
runId: run.id,
|
||||
batchItems: batchItems.length,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (const item of batchItems) {
|
||||
// Don't do anything if this is a batchTriggerAndWait in a deployed task
|
||||
// As that is being handled in resumeDependentParents and resumeTaskRunDependencies
|
||||
if (environment.type !== "DEVELOPMENT" && item.batchTaskRun.dependentTaskAttemptId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.batchTaskRun.batchVersion === "v3") {
|
||||
await completeBatchTaskRunItemV3(item.id, item.batchTaskRunId, this._prisma);
|
||||
} else {
|
||||
// THIS IS DEPRECATED and only happens with batchVersion != v3
|
||||
await this._prisma.batchTaskRunItem.update({
|
||||
where: {
|
||||
id: item.id,
|
||||
},
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
},
|
||||
});
|
||||
|
||||
// This won't resume because this batch does not have a dependent task attempt ID
|
||||
// or is in development, but this service will mark the batch as completed
|
||||
await ResumeBatchRunService.enqueue(item.batchTaskRunId, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async finalizeAttempt({
|
||||
attemptStatus,
|
||||
error,
|
||||
run,
|
||||
}: {
|
||||
attemptStatus?: FINAL_ATTEMPT_STATUSES;
|
||||
error?: TaskRunError;
|
||||
run: TaskRun;
|
||||
}) {
|
||||
if (!attemptStatus && !error) {
|
||||
logger.error("FinalizeTaskRunService: No attemptStatus or error provided", { runId: run.id });
|
||||
return;
|
||||
}
|
||||
|
||||
const latestAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: { taskRunId: run.id },
|
||||
orderBy: { id: "desc" },
|
||||
take: 1,
|
||||
});
|
||||
|
||||
if (latestAttempt) {
|
||||
logger.debug("Finalizing run attempt", {
|
||||
id: latestAttempt.id,
|
||||
status: attemptStatus,
|
||||
error,
|
||||
});
|
||||
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: { id: latestAttempt.id },
|
||||
data: { status: attemptStatus, error: error ? sanitizeError(error) : undefined },
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// There's no attempt, so create one
|
||||
|
||||
logger.debug("Finalizing run no attempt found", {
|
||||
runId: run.id,
|
||||
attemptStatus,
|
||||
error,
|
||||
});
|
||||
|
||||
if (!run.lockedById) {
|
||||
// This happens when a run is expired or was cancelled before an attempt, it's not a problem
|
||||
logger.info(
|
||||
"FinalizeTaskRunService: No lockedById, so can't get the BackgroundWorkerTask. Not creating an attempt.",
|
||||
{ runId: run.id, status: run.status }
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const workerTask = await this._prisma.backgroundWorkerTask.findFirst({
|
||||
select: {
|
||||
id: true,
|
||||
workerId: true,
|
||||
runtimeEnvironmentId: true,
|
||||
queueConfig: true,
|
||||
},
|
||||
where: {
|
||||
id: run.lockedById,
|
||||
},
|
||||
});
|
||||
|
||||
if (!workerTask) {
|
||||
logger.error("FinalizeTaskRunService: No worker task found", { runId: run.id });
|
||||
return;
|
||||
}
|
||||
|
||||
const queue = await findQueueInEnvironment(
|
||||
run.queue,
|
||||
workerTask.runtimeEnvironmentId,
|
||||
workerTask.id,
|
||||
workerTask
|
||||
);
|
||||
|
||||
if (!queue) {
|
||||
logger.error("FinalizeTaskRunService: No queue found", { runId: run.id });
|
||||
return;
|
||||
}
|
||||
|
||||
await this._prisma.taskRunAttempt.create({
|
||||
data: {
|
||||
number: 1,
|
||||
friendlyId: generateFriendlyId("attempt"),
|
||||
taskRunId: run.id,
|
||||
backgroundWorkerId: workerTask?.workerId,
|
||||
backgroundWorkerTaskId: workerTask?.id,
|
||||
queueId: queue.id,
|
||||
runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
|
||||
status: attemptStatus,
|
||||
error: error ? sanitizeError(error) : undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import {
|
||||
type BuildServerMetadata,
|
||||
type InitializeDeploymentRequestBody,
|
||||
type ExternalBuildData,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { customAlphabet } from "nanoid";
|
||||
import { env } from "~/env.server";
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
||||
import { getDeploymentImageRef } from "../getDeploymentImageRef.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { getRegistryConfig } from "../registryConfig.server";
|
||||
import { DeploymentService } from "./deployment.server";
|
||||
import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server";
|
||||
import { errAsync } from "neverthrow";
|
||||
|
||||
const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8);
|
||||
|
||||
export class InitializeDeploymentService extends BaseService {
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
payload: InitializeDeploymentRequestBody
|
||||
) {
|
||||
return this.traceWithEnv("call", environment, async () => {
|
||||
if (payload.gitMeta?.commitSha?.startsWith("deployment_")) {
|
||||
// When we introduced automatic deployments via the build server, we slightly changed the deployment flow
|
||||
// mainly in the initialization and starting step: now deployments are first initialized in the `PENDING` status
|
||||
// and updated to `BUILDING` once the build server dequeues the build job.
|
||||
// Newer versions of the `deploy` command in the CLI will automatically attach to the existing deployment
|
||||
// and continue with the build process. For older versions, we can't change the command's client-side behavior,
|
||||
// so we need to handle this case here in the initialization endpoint. As we control the env variables which
|
||||
// the git meta is extracted from in the build server, we can use those to pass the existing deployment ID
|
||||
// to this endpoint. This doesn't affect the git meta on the deployment as it is set prior to this step using the
|
||||
// /start endpoint. It's a rather hacky solution, but it will do for now as it enables us to avoid degrading the
|
||||
// build server experience for users with older CLI versions. We'll eventually be able to remove this workaround
|
||||
// once we stop supporting 3.x CLI versions.
|
||||
|
||||
const existingDeploymentId = payload.gitMeta.commitSha;
|
||||
const existingDeployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: {
|
||||
environmentId: environment.id,
|
||||
friendlyId: existingDeploymentId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!existingDeployment) {
|
||||
throw new ServiceValidationError(
|
||||
"Existing deployment not found during deployment initialization"
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
deployment: existingDeployment,
|
||||
imageRef: existingDeployment.imageReference ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
// v4 CLI versions always send `payload.type` ("MANAGED" or "V1"). v3 CLI
|
||||
// versions never do, so the absence of `type` is a reliable signal that
|
||||
// the request came from a 3.x CLI. Detection always runs (so we can
|
||||
// observe how many deploys are still using v3), enforcement is gated
|
||||
// behind DEPRECATE_V3_CLI_DEPLOYS_ENABLED so it can be rolled out safely.
|
||||
if (!payload.type) {
|
||||
const enforced = env.DEPRECATE_V3_CLI_DEPLOYS_ENABLED === "1";
|
||||
|
||||
logger.warn("Detected deploy from deprecated v3 CLI", {
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.project.organizationId,
|
||||
enforced,
|
||||
});
|
||||
|
||||
if (enforced) {
|
||||
throw new ServiceValidationError(
|
||||
"The trigger.dev CLI v3 is no longer supported for deployments. Please upgrade your project to v4: https://trigger.dev/docs/migrating-from-v3"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (payload.type === "UNMANAGED") {
|
||||
throw new ServiceValidationError("UNMANAGED deployments are not supported");
|
||||
}
|
||||
|
||||
// Upgrade the project to engine "V2" if it's not already. This should cover cases where people deploy to V2 without running dev first.
|
||||
if (payload.type === "MANAGED" && environment.project.engine === "V1") {
|
||||
await this._prisma.project.update({
|
||||
where: {
|
||||
id: environment.project.id,
|
||||
},
|
||||
data: {
|
||||
engine: "V2",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (payload.selfHosted && remoteBuildsEnabled()) {
|
||||
throw new ServiceValidationError(
|
||||
"Self-hosted deployments are not supported on this instance"
|
||||
);
|
||||
}
|
||||
|
||||
// For the `PENDING` initial status, defer the creation of the Depot build until the deployment is started to avoid token expiration issues.
|
||||
// For local and native builds we don't need to generate the Depot tokens. We still need to create an empty object sadly due to a bug in older CLI versions.
|
||||
const generateExternalBuildToken =
|
||||
payload.initialStatus === "PENDING" || payload.isNativeBuild || payload.isLocalBuild;
|
||||
|
||||
const externalBuildData = generateExternalBuildToken
|
||||
? ({
|
||||
projectId: "-",
|
||||
buildToken: "-",
|
||||
buildId: "-",
|
||||
} satisfies ExternalBuildData)
|
||||
: await createRemoteImageBuild(environment.project);
|
||||
|
||||
const triggeredBy = payload.userId
|
||||
? await this._prisma.user.findFirst({
|
||||
where: {
|
||||
id: payload.userId,
|
||||
orgMemberships: {
|
||||
some: {
|
||||
organizationId: environment.project.organizationId,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const isV4Deployment = payload.type === "MANAGED";
|
||||
const registryConfig = getRegistryConfig(isV4Deployment);
|
||||
|
||||
const deploymentShortCode = nanoid(8);
|
||||
|
||||
// We keep using `BUILDING` as the initial status if not explicitly set
|
||||
// to avoid changing the behavior for deployments not created in the build server.
|
||||
// Native builds always start in the `PENDING` status.
|
||||
const initialStatus =
|
||||
payload.initialStatus ?? (payload.isNativeBuild ? "PENDING" : "BUILDING");
|
||||
|
||||
const deploymentService = new DeploymentService();
|
||||
const s2StreamOrFail = await deploymentService
|
||||
.createEventStream(environment.project, { shortCode: deploymentShortCode })
|
||||
.andThen(({ basin, stream }) =>
|
||||
deploymentService.getEventStreamAccessToken(environment.project).map((accessToken) => ({
|
||||
basin,
|
||||
stream,
|
||||
accessToken,
|
||||
}))
|
||||
);
|
||||
|
||||
if (s2StreamOrFail.isErr()) {
|
||||
logger.error(
|
||||
"Failed to create S2 event stream on deployment initialization, continuing without logs stream",
|
||||
{
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
error: s2StreamOrFail.error,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const eventStream = s2StreamOrFail.isOk()
|
||||
? {
|
||||
s2: {
|
||||
basin: s2StreamOrFail.value.basin,
|
||||
stream: s2StreamOrFail.value.stream,
|
||||
accessToken: s2StreamOrFail.value.accessToken,
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const buildServerMetadata: BuildServerMetadata | undefined =
|
||||
payload.isNativeBuild || payload.buildId
|
||||
? {
|
||||
buildId: payload.buildId,
|
||||
...(payload.isNativeBuild
|
||||
? {
|
||||
isNativeBuild: payload.isNativeBuild,
|
||||
artifactKey: payload.artifactKey,
|
||||
skipPromotion: payload.skipPromotion,
|
||||
configFilePath: payload.configFilePath,
|
||||
skipEnqueue: payload.skipEnqueue,
|
||||
}
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
|
||||
// Concurrent deploys to the same environment race on the
|
||||
// `(environmentId, version)` unique constraint. The helper retries on
|
||||
// P2002, recomputing the version (and re-running the image ref call so
|
||||
// the persisted imageReference always matches the persisted version)
|
||||
// each attempt.
|
||||
const deployment = await createDeploymentWithNextVersion(
|
||||
this._prisma,
|
||||
environment.id,
|
||||
async (nextVersion) => {
|
||||
const [imageRefError, imageRefResult] = await tryCatch(
|
||||
getDeploymentImageRef({
|
||||
registry: registryConfig,
|
||||
projectRef: environment.project.externalRef,
|
||||
nextVersion,
|
||||
environmentType: environment.type,
|
||||
deploymentShortCode,
|
||||
})
|
||||
);
|
||||
|
||||
if (imageRefError) {
|
||||
logger.error("Failed to get deployment image ref", {
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
version: nextVersion,
|
||||
triggeredById: triggeredBy?.id,
|
||||
type: payload.type,
|
||||
cause: imageRefError.message,
|
||||
});
|
||||
throw new ServiceValidationError("Failed to get deployment image ref");
|
||||
}
|
||||
|
||||
const { imageRef, isEcr, repoCreated } = imageRefResult;
|
||||
|
||||
logger.debug("Creating deployment", {
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
version: nextVersion,
|
||||
triggeredById: triggeredBy?.id,
|
||||
type: payload.type,
|
||||
imageRef,
|
||||
isEcr,
|
||||
repoCreated,
|
||||
initialStatus,
|
||||
artifactKey: payload.isNativeBuild ? payload.artifactKey : undefined,
|
||||
isNativeBuild: payload.isNativeBuild,
|
||||
});
|
||||
|
||||
return {
|
||||
// Regenerated per attempt: each attempt is a fresh `create` that
|
||||
// must satisfy `WorkerDeployment.friendlyId @unique`, so reusing a
|
||||
// friendlyId across retries would risk a spurious P2002 on
|
||||
// friendlyId instead of the version collision we're retrying.
|
||||
friendlyId: generateFriendlyId("deployment"),
|
||||
contentHash: payload.contentHash,
|
||||
shortCode: deploymentShortCode,
|
||||
status: initialStatus,
|
||||
projectId: environment.projectId,
|
||||
externalBuildData,
|
||||
buildServerMetadata,
|
||||
triggeredById: triggeredBy?.id,
|
||||
type: payload.type,
|
||||
imageReference: imageRef,
|
||||
imagePlatform: env.DEPLOY_IMAGE_PLATFORM,
|
||||
git: payload.gitMeta ?? undefined,
|
||||
commitSHA: payload.gitMeta?.commitSha ?? undefined,
|
||||
runtime: payload.runtime ?? undefined,
|
||||
triggeredVia: payload.triggeredVia ?? undefined,
|
||||
startedAt: initialStatus === "BUILDING" ? new Date() : undefined,
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const timeoutMs =
|
||||
deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS;
|
||||
|
||||
await TimeoutDeploymentService.enqueue(
|
||||
deployment.id,
|
||||
deployment.status,
|
||||
"Building timed out",
|
||||
new Date(Date.now() + timeoutMs)
|
||||
);
|
||||
|
||||
// For github integration there is no artifactKey, hence we skip it here
|
||||
if (payload.isNativeBuild && payload.artifactKey && !payload.skipEnqueue) {
|
||||
const result = await deploymentService
|
||||
.enqueueBuild(environment, deployment, payload.artifactKey, {
|
||||
skipPromotion: payload.skipPromotion,
|
||||
configFilePath: payload.configFilePath,
|
||||
})
|
||||
.orElse((error) => {
|
||||
logger.error("Failed to enqueue build", {
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
deploymentId: deployment.id,
|
||||
error: error.cause,
|
||||
});
|
||||
|
||||
return deploymentService
|
||||
.cancelDeployment(environment, deployment.friendlyId, {
|
||||
canceledReason: "Failed to enqueue build, please try again shortly.",
|
||||
})
|
||||
.orTee((cancelError) =>
|
||||
logger.error("Failed to cancel deployment after failed build enqueue", {
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
deploymentId: deployment.id,
|
||||
error: cancelError,
|
||||
})
|
||||
)
|
||||
.andThen(() => errAsync(error))
|
||||
.orElse(() => errAsync(error));
|
||||
});
|
||||
|
||||
if (result.isErr()) {
|
||||
throw Error("Failed to enqueue build");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
deployment,
|
||||
imageRef: deployment.imageReference ?? "",
|
||||
eventStream,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
import {
|
||||
isUniqueConstraintError,
|
||||
type Prisma,
|
||||
type PrismaClientOrTransaction,
|
||||
type WorkerDeployment,
|
||||
} from "@trigger.dev/database";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { calculateNextBuildVersion } from "../../utils/calculateNextBuildVersion";
|
||||
|
||||
export type CreateDeploymentData = Omit<
|
||||
Prisma.WorkerDeploymentUncheckedCreateInput,
|
||||
"version" | "environmentId"
|
||||
>;
|
||||
|
||||
export type CreateDeploymentWithNextVersionOptions = {
|
||||
maxRetries?: number;
|
||||
jitterMs?: { min: number; max: number };
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_RETRIES = 5;
|
||||
const DEFAULT_JITTER_MS = { min: 5, max: 50 };
|
||||
|
||||
export class DeploymentVersionCollisionError extends Error {
|
||||
readonly name = "DeploymentVersionCollisionError";
|
||||
readonly environmentId: string;
|
||||
readonly attempts: number;
|
||||
readonly lastAttemptedVersion: string;
|
||||
|
||||
constructor(args: {
|
||||
environmentId: string;
|
||||
attempts: number;
|
||||
lastAttemptedVersion: string;
|
||||
cause: unknown;
|
||||
}) {
|
||||
super(
|
||||
`Failed to allocate a unique worker deployment version for environment ${args.environmentId} after ${args.attempts} attempt(s); last tried "${args.lastAttemptedVersion}"`,
|
||||
{ cause: args.cause }
|
||||
);
|
||||
this.environmentId = args.environmentId;
|
||||
this.attempts = args.attempts;
|
||||
this.lastAttemptedVersion = args.lastAttemptedVersion;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createDeploymentWithNextVersion(
|
||||
prisma: PrismaClientOrTransaction,
|
||||
environmentId: string,
|
||||
buildData: (nextVersion: string) => CreateDeploymentData | Promise<CreateDeploymentData>,
|
||||
options: CreateDeploymentWithNextVersionOptions = {}
|
||||
): Promise<WorkerDeployment> {
|
||||
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
||||
const jitterMs = options.jitterMs ?? DEFAULT_JITTER_MS;
|
||||
|
||||
let lastError: unknown;
|
||||
let lastVersion = "";
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
const latest = await prisma.workerDeployment.findFirst({
|
||||
where: { environmentId },
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 1,
|
||||
});
|
||||
|
||||
const version = calculateNextBuildVersion(latest?.version);
|
||||
lastVersion = version;
|
||||
const data = await buildData(version);
|
||||
|
||||
try {
|
||||
return await prisma.workerDeployment.create({
|
||||
data: { ...data, environmentId, version },
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isUniqueConstraintError(error, ["environmentId", "version"])) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
lastError = error;
|
||||
logger.warn("Worker deployment version collided, retrying", {
|
||||
environmentId,
|
||||
attempt: attempt + 1,
|
||||
maxRetries,
|
||||
attemptedVersion: version,
|
||||
});
|
||||
|
||||
// Randomised backoff so N concurrent racers don't loop in lockstep into the
|
||||
// same collision again.
|
||||
const delay = jitterMs.min + Math.random() * (jitterMs.max - jitterMs.min);
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
throw new DeploymentVersionCollisionError({
|
||||
environmentId,
|
||||
attempts: maxRetries + 1,
|
||||
lastAttemptedVersion: lastVersion,
|
||||
cause: lastError,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { EnvironmentPauseSource, type PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import { prisma } from "~/db.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getManualPauseEnvironmentResult } from "~/v3/services/billingLimit/manualPauseEnvironmentGuard.server";
|
||||
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
||||
import { WithRunEngine } from "./baseService.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
|
||||
export type PauseStatus = "paused" | "resumed";
|
||||
|
||||
export type PauseEnvironmentResult =
|
||||
| {
|
||||
success: true;
|
||||
state: PauseStatus;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export class PauseEnvironmentService extends WithRunEngine {
|
||||
constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {
|
||||
super({ prisma });
|
||||
}
|
||||
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
action: PauseStatus
|
||||
): Promise<PauseEnvironmentResult> {
|
||||
try {
|
||||
const org = await this._prisma.organization.findFirst({
|
||||
where: {
|
||||
id: environment.organizationId,
|
||||
},
|
||||
select: {
|
||||
runsEnabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!org) {
|
||||
throw new Error("Organization not found");
|
||||
}
|
||||
|
||||
const previousPauseState = await this._prisma.runtimeEnvironment.findFirst({
|
||||
where: { id: environment.id },
|
||||
select: {
|
||||
paused: true,
|
||||
pauseSource: true,
|
||||
},
|
||||
});
|
||||
|
||||
const manualPauseGuard = getManualPauseEnvironmentResult(
|
||||
action,
|
||||
previousPauseState?.pauseSource
|
||||
);
|
||||
if (!manualPauseGuard.proceed) {
|
||||
if (manualPauseGuard.success) {
|
||||
return {
|
||||
success: true,
|
||||
state: manualPauseGuard.state,
|
||||
};
|
||||
}
|
||||
// Expected, user-actionable guard result, not an error: return it as a failure
|
||||
// result so it doesn't reach Sentry via the catch below.
|
||||
return {
|
||||
success: false,
|
||||
error: manualPauseGuard.error,
|
||||
};
|
||||
}
|
||||
|
||||
if (!org.runsEnabled && action === "resumed") {
|
||||
throw new Error(
|
||||
"Runs are disabled for this organization. Your free plan has probably been exceeded. If not please contact support."
|
||||
);
|
||||
}
|
||||
|
||||
if (action === "resumed") {
|
||||
const resumed = await this._prisma.runtimeEnvironment.updateMany({
|
||||
where: {
|
||||
id: environment.id,
|
||||
// NOT on a nullable field excludes NULL rows in Prisma, which made
|
||||
// user-paused envs (pauseSource null) unresumable.
|
||||
OR: [
|
||||
{ pauseSource: null },
|
||||
{ NOT: { pauseSource: EnvironmentPauseSource.BILLING_LIMIT } },
|
||||
],
|
||||
},
|
||||
data: {
|
||||
paused: false,
|
||||
pauseSource: null,
|
||||
},
|
||||
});
|
||||
|
||||
if (resumed.count === 0) {
|
||||
// Raced into the paused state after the guard read above: expected,
|
||||
// return as a failure result rather than throwing to Sentry.
|
||||
return {
|
||||
success: false,
|
||||
error:
|
||||
"This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
await this._prisma.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: { paused: true },
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (action === "paused") {
|
||||
logger.debug("PauseEnvironmentService: pausing environment", {
|
||||
environmentId: environment.id,
|
||||
});
|
||||
await updateEnvConcurrencyLimits(environment, 0);
|
||||
} else {
|
||||
logger.debug("PauseEnvironmentService: resuming environment", {
|
||||
environmentId: environment.id,
|
||||
});
|
||||
await updateEnvConcurrencyLimits(environment);
|
||||
}
|
||||
} catch (error) {
|
||||
await this._prisma.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: previousPauseState?.paused ?? action === "resumed",
|
||||
pauseSource: previousPauseState?.pauseSource ?? null,
|
||||
},
|
||||
});
|
||||
// Rollback still wrote the env row; drop any cached copy before rethrowing.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// The env's `paused` state changed in the control-plane; drop any cached copy.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
state: action,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error("PauseEnvironmentService: error pausing environment", {
|
||||
action,
|
||||
environmentId: environment.id,
|
||||
error,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { type QueueItem, type RetrieveQueueParam } from "@trigger.dev/core/v3";
|
||||
import { getQueue, toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server";
|
||||
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { determineEngineVersion } from "../engineVersion.server";
|
||||
import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server";
|
||||
import { engine } from "../runEngine.server";
|
||||
|
||||
export type PauseStatus = "paused" | "resumed";
|
||||
|
||||
export type PauseQueueResult =
|
||||
| {
|
||||
success: true;
|
||||
state: PauseStatus;
|
||||
queue: QueueItem;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
code: "queue-not-found" | "unknown-error" | "engine-version";
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export class PauseQueueService extends BaseService {
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
queueInput: RetrieveQueueParam,
|
||||
action: PauseStatus
|
||||
): Promise<PauseQueueResult> {
|
||||
try {
|
||||
//check the engine is the correct version
|
||||
const engineVersion = await determineEngineVersion({ environment });
|
||||
|
||||
if (engineVersion === "V1") {
|
||||
return {
|
||||
success: false as const,
|
||||
code: "engine-version",
|
||||
error: "Upgrade to v4+ to pause/resume queues",
|
||||
};
|
||||
}
|
||||
|
||||
const queue = await getQueue(this._prisma, environment, queueInput);
|
||||
|
||||
if (!queue) {
|
||||
return {
|
||||
success: false,
|
||||
code: "queue-not-found",
|
||||
};
|
||||
}
|
||||
|
||||
const updatedQueue = await this._prisma.taskQueue.update({
|
||||
where: {
|
||||
id: queue.id,
|
||||
},
|
||||
data: {
|
||||
paused: action === "paused",
|
||||
},
|
||||
});
|
||||
|
||||
if (action === "paused") {
|
||||
await updateQueueConcurrencyLimits(environment, queue.name, 0);
|
||||
} else {
|
||||
if (queue.concurrencyLimit) {
|
||||
await updateQueueConcurrencyLimits(environment, queue.name, queue.concurrencyLimit);
|
||||
} else {
|
||||
await removeQueueConcurrencyLimits(environment, queue.name);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("PauseQueueService: queue state updated", {
|
||||
queueId: queue.id,
|
||||
action,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
|
||||
const results = await Promise.all([
|
||||
engine.lengthOfQueues(environment, [queue.name]),
|
||||
engine.currentConcurrencyOfQueues(environment, [queue.name]),
|
||||
]);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
state: action,
|
||||
queue: toQueueItem({
|
||||
friendlyId: updatedQueue.friendlyId,
|
||||
name: updatedQueue.name,
|
||||
type: updatedQueue.type,
|
||||
running: results[1]?.[updatedQueue.name] ?? 0,
|
||||
queued: results[0]?.[updatedQueue.name] ?? 0,
|
||||
concurrencyLimit: updatedQueue.concurrencyLimit ?? null,
|
||||
concurrencyLimitBase: updatedQueue.concurrencyLimitBase ?? null,
|
||||
concurrencyLimitOverriddenAt: updatedQueue.concurrencyLimitOverriddenAt ?? null,
|
||||
concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null,
|
||||
paused: updatedQueue.paused,
|
||||
}),
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error("PauseQueueService: error updating queue state", {
|
||||
error,
|
||||
environmentId: environment.id,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
code: "unknown-error",
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { z } from "zod";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import type { ZodSubscriber } from "../utils/zodPubSub.server";
|
||||
import { ZodPubSub } from "../utils/zodPubSub.server";
|
||||
import { env } from "~/env.server";
|
||||
import { Gauge } from "prom-client";
|
||||
import { metricsRegister } from "~/metrics.server";
|
||||
|
||||
const messageCatalog = {
|
||||
WORKER_CREATED: z.object({
|
||||
environmentId: z.string(),
|
||||
environmentType: z.string(),
|
||||
createdAt: z.coerce.date(),
|
||||
taskCount: z.number(),
|
||||
type: z.union([z.literal("local"), z.literal("deployed")]),
|
||||
}),
|
||||
};
|
||||
|
||||
export type ProjectSubscriber = ZodSubscriber<typeof messageCatalog>;
|
||||
|
||||
export const projectPubSub = singleton("projectPubSub", initializeProjectPubSub);
|
||||
|
||||
function initializeProjectPubSub() {
|
||||
const pubSub = new ZodPubSub({
|
||||
redis: {
|
||||
port: env.PUBSUB_REDIS_PORT,
|
||||
host: env.PUBSUB_REDIS_HOST,
|
||||
username: env.PUBSUB_REDIS_USERNAME,
|
||||
password: env.PUBSUB_REDIS_PASSWORD,
|
||||
tlsDisabled: env.PUBSUB_REDIS_TLS_DISABLED === "true",
|
||||
clusterMode: env.PUBSUB_REDIS_CLUSTER_MODE_ENABLED === "1",
|
||||
},
|
||||
schema: messageCatalog,
|
||||
});
|
||||
|
||||
new Gauge({
|
||||
name: "project_pub_sub_subscribers",
|
||||
help: "Number of project pub sub subscribers",
|
||||
collect() {
|
||||
this.set(pubSub.subscriberCount);
|
||||
},
|
||||
registers: [metricsRegister],
|
||||
});
|
||||
|
||||
return pubSub;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// `source: "code"` is reserved for the deploy path; no other caller may write
|
||||
// it. Normalize anything that isn't a legitimate caller-supplied value to
|
||||
// "dashboard". Dependency-free so the rule can be unit-tested directly; it
|
||||
// backs the service-layer check (the route layer also constrains `source`).
|
||||
export function normalizePromptOverrideSource(source: string | undefined | null): string {
|
||||
return source && source !== "code" ? source : "dashboard";
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import { createHash } from "crypto";
|
||||
import { prisma } from "~/db.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { normalizePromptOverrideSource } from "./promptOverrideSource";
|
||||
|
||||
export class PromptService extends BaseService {
|
||||
async promoteVersion(promptId: string, versionId: string, options?: { sourceGuard?: boolean }) {
|
||||
const target = await this._prisma.promptVersion.findUnique({
|
||||
where: { id: versionId },
|
||||
});
|
||||
|
||||
if (!target) {
|
||||
throw new ServiceValidationError("Version not found", 404);
|
||||
}
|
||||
|
||||
if (target.promptId !== promptId) {
|
||||
throw new ServiceValidationError("Version does not belong to this prompt", 400);
|
||||
}
|
||||
|
||||
if (options?.sourceGuard && target.source !== "code") {
|
||||
throw new ServiceValidationError(
|
||||
"Only code-sourced versions can be promoted. Use the override API instead.",
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
await prisma.$transaction(async (tx) => {
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", 'current')
|
||||
WHERE "promptId" = ${promptId} AND 'current' = ANY("labels")
|
||||
`;
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_append("labels", 'current')
|
||||
WHERE "id" = ${versionId} AND NOT ('current' = ANY("labels"))
|
||||
`;
|
||||
});
|
||||
}
|
||||
|
||||
async createOverride(
|
||||
promptId: string,
|
||||
data: {
|
||||
textContent: string;
|
||||
model?: string;
|
||||
commitMessage?: string;
|
||||
source?: string;
|
||||
createdBy?: string;
|
||||
}
|
||||
) {
|
||||
const contentHash = createHash("sha256").update(data.textContent).digest("hex").slice(0, 16);
|
||||
|
||||
const result = await prisma.$transaction(async (tx) => {
|
||||
const latest = await tx.promptVersion.findFirst({
|
||||
where: { promptId },
|
||||
orderBy: { version: "desc" },
|
||||
});
|
||||
const nextVersion = (latest?.version ?? 0) + 1;
|
||||
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", 'override')
|
||||
WHERE "promptId" = ${promptId} AND 'override' = ANY("labels")
|
||||
`;
|
||||
|
||||
// Defence in depth: reject the reserved `code` source regardless of
|
||||
// caller, in case a future caller skips the route-layer check.
|
||||
const safeSource = normalizePromptOverrideSource(data.source);
|
||||
|
||||
await tx.promptVersion.create({
|
||||
data: {
|
||||
promptId,
|
||||
version: nextVersion,
|
||||
textContent: data.textContent,
|
||||
model: data.model || null,
|
||||
source: safeSource,
|
||||
commitMessage: data.commitMessage || null,
|
||||
contentHash,
|
||||
labels: ["override"],
|
||||
createdBy: data.createdBy,
|
||||
},
|
||||
});
|
||||
|
||||
return { version: nextVersion };
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async updateOverride(
|
||||
promptId: string,
|
||||
data: {
|
||||
textContent?: string;
|
||||
model?: string;
|
||||
commitMessage?: string;
|
||||
}
|
||||
) {
|
||||
const overrideVer = await this._prisma.promptVersion.findFirst({
|
||||
where: { promptId, labels: { has: "override" } },
|
||||
orderBy: { version: "desc" },
|
||||
});
|
||||
|
||||
if (!overrideVer) {
|
||||
throw new ServiceValidationError("No active override to update", 400);
|
||||
}
|
||||
|
||||
const contentString = data.textContent ?? overrideVer.textContent ?? "";
|
||||
const contentHash = createHash("sha256").update(contentString).digest("hex").slice(0, 16);
|
||||
|
||||
await this._prisma.promptVersion.update({
|
||||
where: { id: overrideVer.id },
|
||||
data: {
|
||||
textContent: contentString,
|
||||
model: data.model ?? overrideVer.model,
|
||||
commitMessage: data.commitMessage ?? overrideVer.commitMessage,
|
||||
contentHash,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async removeOverride(promptId: string) {
|
||||
await this.#removeLabel(promptId, "override");
|
||||
}
|
||||
|
||||
async reactivateOverride(promptId: string, versionId: string) {
|
||||
const target = await this._prisma.promptVersion.findUnique({
|
||||
where: { id: versionId },
|
||||
});
|
||||
|
||||
if (!target) {
|
||||
throw new ServiceValidationError("Version not found", 404);
|
||||
}
|
||||
|
||||
if (target.promptId !== promptId) {
|
||||
throw new ServiceValidationError("Version does not belong to this prompt", 400);
|
||||
}
|
||||
|
||||
if (target.source === "code") {
|
||||
throw new ServiceValidationError(
|
||||
"Code-sourced versions cannot be reactivated as overrides",
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
await prisma.$transaction(async (tx) => {
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", 'override')
|
||||
WHERE "promptId" = ${promptId} AND 'override' = ANY("labels")
|
||||
`;
|
||||
await tx.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_append("labels", 'override')
|
||||
WHERE "id" = ${versionId} AND NOT ('override' = ANY("labels"))
|
||||
`;
|
||||
});
|
||||
}
|
||||
|
||||
async #removeLabel(promptId: string, label: string) {
|
||||
await this._prisma.$executeRaw`
|
||||
UPDATE "prompt_versions"
|
||||
SET "labels" = array_remove("labels", ${label})
|
||||
WHERE "promptId" = ${promptId} AND ${label} = ANY("labels")
|
||||
`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { SpanStatusCode } from "@opentelemetry/api";
|
||||
import { type WorkerDeploymentStatus } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { tracer } from "~/v3/tracer.server";
|
||||
|
||||
type TerminalDeploymentStatus = Extract<
|
||||
WorkerDeploymentStatus,
|
||||
"DEPLOYED" | "FAILED" | "TIMED_OUT"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Records a deployment's terminal status as a `deployment.outcome` span so
|
||||
* deploy success/failure is queryable from traces (no DB read). Call after each
|
||||
* terminal-status write. Org/project/env are best-effort; never throws.
|
||||
*/
|
||||
export function recordDeploymentOutcome(params: {
|
||||
status: TerminalDeploymentStatus;
|
||||
deploymentFriendlyId: string;
|
||||
organizationId?: string;
|
||||
projectId?: string;
|
||||
environmentId?: string;
|
||||
environmentType?: string;
|
||||
reason?: string;
|
||||
}): void {
|
||||
try {
|
||||
const span = tracer.startSpan("deployment.outcome", {
|
||||
attributes: {
|
||||
"$trigger.org.id": params.organizationId,
|
||||
"$trigger.project.id": params.projectId,
|
||||
"$trigger.env.id": params.environmentId,
|
||||
"$trigger.env.type": params.environmentType,
|
||||
"deployment.outcome.status": params.status,
|
||||
"deployment.outcome.success": params.status === "DEPLOYED",
|
||||
"deployment.outcome.deployment_id": params.deploymentFriendlyId,
|
||||
"deployment.outcome.reason": params.reason,
|
||||
},
|
||||
});
|
||||
|
||||
if (params.status !== "DEPLOYED") {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason });
|
||||
}
|
||||
|
||||
span.end();
|
||||
} catch (error) {
|
||||
logger.debug("recordDeploymentOutcome failed", {
|
||||
deploymentFriendlyId: params.deploymentFriendlyId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import {
|
||||
type MachinePresetName,
|
||||
conditionallyImportPacket,
|
||||
parsePacket,
|
||||
stringifyIO,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { type TaskRun } from "@trigger.dev/database";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import { baseWorkerQueue } from "~/runEngine/concerns/workerQueueSplit.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { OutOfEntitlementError, TriggerTaskService } from "./triggerTask.server";
|
||||
import { type RunOptionsData } from "../testTask";
|
||||
import { replaceSuperJsonPayload } from "@trigger.dev/core/v3/utils/ioSerialization";
|
||||
import { determineRealtimeStreamsVersion } from "~/services/realtime/v1StreamsGlobal.server";
|
||||
|
||||
type OverrideOptions = {
|
||||
environmentId?: string;
|
||||
payload?: string;
|
||||
metadata?: unknown;
|
||||
bulkActionId?: string;
|
||||
triggerSource?: string;
|
||||
} & RunOptionsData;
|
||||
|
||||
export class ReplayTaskRunService extends BaseService {
|
||||
public async call(existingTaskRun: TaskRun, overrideOptions: OverrideOptions = {}) {
|
||||
// An override environment must belong to the same project as the source
|
||||
// run. The source project is derived from the run's own environment rather
|
||||
// than existingTaskRun.projectId, since the buffered-run fallback passes a
|
||||
// synthetic TaskRun with no projectId. Only check when a distinct override
|
||||
// is supplied; otherwise the run's own environment is used.
|
||||
if (
|
||||
overrideOptions.environmentId &&
|
||||
overrideOptions.environmentId !== existingTaskRun.runtimeEnvironmentId
|
||||
) {
|
||||
const [overrideEnvironment, sourceEnvironment] = await Promise.all([
|
||||
this._prisma.runtimeEnvironment.findFirst({
|
||||
where: { id: overrideOptions.environmentId },
|
||||
select: { projectId: true },
|
||||
}),
|
||||
this._prisma.runtimeEnvironment.findFirst({
|
||||
where: { id: existingTaskRun.runtimeEnvironmentId },
|
||||
select: { projectId: true },
|
||||
}),
|
||||
]);
|
||||
if (
|
||||
!overrideEnvironment ||
|
||||
!sourceEnvironment ||
|
||||
overrideEnvironment.projectId !== sourceEnvironment.projectId
|
||||
) {
|
||||
logger.warn("Refusing to replay a run into an environment outside its project", {
|
||||
taskRunId: existingTaskRun.id,
|
||||
taskRunFriendlyId: existingTaskRun.friendlyId,
|
||||
sourceEnvironmentId: existingTaskRun.runtimeEnvironmentId,
|
||||
sourceProjectId: sourceEnvironment?.projectId ?? null,
|
||||
overrideEnvironmentId: overrideOptions.environmentId,
|
||||
overrideProjectId: overrideEnvironment?.projectId ?? null,
|
||||
});
|
||||
throw new Error("Cannot replay a run into an environment outside its project");
|
||||
}
|
||||
}
|
||||
|
||||
const authenticatedEnvironment = await findEnvironmentById(
|
||||
overrideOptions.environmentId ?? existingTaskRun.runtimeEnvironmentId
|
||||
);
|
||||
if (!authenticatedEnvironment) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (authenticatedEnvironment.archivedAt) {
|
||||
throw new Error("Can't replay a run on an archived environment");
|
||||
}
|
||||
|
||||
logger.info("Replaying task run", {
|
||||
taskRunId: existingTaskRun.id,
|
||||
taskRunFriendlyId: existingTaskRun.friendlyId,
|
||||
});
|
||||
|
||||
const existingEnvironment = await this._prisma.runtimeEnvironment.findFirstOrThrow({
|
||||
where: {
|
||||
id: existingTaskRun.runtimeEnvironmentId,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
},
|
||||
});
|
||||
|
||||
const payloadPacket = await this.overrideExistingPayloadPacket(
|
||||
existingTaskRun,
|
||||
overrideOptions.payload
|
||||
);
|
||||
const parsedPayload =
|
||||
payloadPacket.dataType === "application/json"
|
||||
? await parsePacket(payloadPacket)
|
||||
: payloadPacket.data;
|
||||
const payloadType = payloadPacket.dataType;
|
||||
const metadata = overrideOptions.metadata ?? (await this.getExistingMetadata(existingTaskRun));
|
||||
const tags = overrideOptions.tags ?? existingTaskRun.runTags;
|
||||
// Only use the region from the existing run if V2 engine and neither environment is dev
|
||||
const ignoreRegion =
|
||||
existingTaskRun.engine === "V1" ||
|
||||
existingEnvironment.type === "DEVELOPMENT" ||
|
||||
authenticatedEnvironment.type === "DEVELOPMENT";
|
||||
const region = ignoreRegion
|
||||
? undefined
|
||||
: overrideOptions.region ||
|
||||
existingTaskRun.region ||
|
||||
baseWorkerQueue(existingTaskRun.workerQueue);
|
||||
|
||||
try {
|
||||
const taskQueue = await this._prisma.taskQueue.findFirst({
|
||||
where: {
|
||||
runtimeEnvironmentId: authenticatedEnvironment.id,
|
||||
name: overrideOptions.queue ?? existingTaskRun.queue,
|
||||
},
|
||||
});
|
||||
|
||||
const triggerTaskService = new TriggerTaskService();
|
||||
const result = await triggerTaskService.call(
|
||||
existingTaskRun.taskIdentifier,
|
||||
authenticatedEnvironment,
|
||||
{
|
||||
payload: parsedPayload,
|
||||
options: {
|
||||
payloadType,
|
||||
queue: taskQueue
|
||||
? {
|
||||
name: taskQueue.name,
|
||||
}
|
||||
: undefined,
|
||||
test: existingTaskRun.isTest,
|
||||
tags,
|
||||
metadata: metadata,
|
||||
delay: overrideOptions.delaySeconds
|
||||
? new Date(Date.now() + overrideOptions.delaySeconds * 1000)
|
||||
: undefined,
|
||||
ttl: overrideOptions.ttlSeconds,
|
||||
idempotencyKey: overrideOptions.idempotencyKey,
|
||||
idempotencyKeyTTL: overrideOptions.idempotencyKeyTTLSeconds
|
||||
? `${overrideOptions.idempotencyKeyTTLSeconds}s`
|
||||
: undefined,
|
||||
concurrencyKey:
|
||||
overrideOptions.concurrencyKey ?? existingTaskRun.concurrencyKey ?? undefined,
|
||||
maxAttempts: overrideOptions.maxAttempts,
|
||||
maxDuration: overrideOptions.maxDurationSeconds,
|
||||
machine:
|
||||
overrideOptions.machine ??
|
||||
(existingTaskRun.machinePreset as MachinePresetName) ??
|
||||
undefined,
|
||||
lockToVersion:
|
||||
overrideOptions.version === "latest" ? undefined : overrideOptions.version,
|
||||
bulkActionId: overrideOptions?.bulkActionId,
|
||||
region,
|
||||
priority: overrideOptions.prioritySeconds,
|
||||
},
|
||||
},
|
||||
{
|
||||
spanParentAsLink: true,
|
||||
parentAsLinkType: "replay",
|
||||
replayedFromTaskRunFriendlyId: existingTaskRun.friendlyId,
|
||||
traceContext: {
|
||||
traceparent: `00-${existingTaskRun.traceId}-${existingTaskRun.spanId}-01`,
|
||||
},
|
||||
realtimeStreamsVersion: determineRealtimeStreamsVersion(
|
||||
existingTaskRun.realtimeStreamsVersion
|
||||
),
|
||||
triggerSource: overrideOptions.triggerSource ?? "api",
|
||||
triggerAction: "replay",
|
||||
}
|
||||
);
|
||||
|
||||
return result?.run;
|
||||
} catch (error) {
|
||||
if (error instanceof OutOfEntitlementError) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error("Failed to replay a run", {
|
||||
error: error instanceof Error ? error.message : error,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private async overrideExistingPayloadPacket(
|
||||
existingTaskRun: TaskRun,
|
||||
stringifiedPayloadOverride: string | undefined
|
||||
) {
|
||||
if (existingTaskRun.payloadType === "application/store") {
|
||||
return conditionallyImportPacket({
|
||||
data: existingTaskRun.payload,
|
||||
dataType: existingTaskRun.payloadType,
|
||||
});
|
||||
}
|
||||
|
||||
if (stringifiedPayloadOverride && existingTaskRun.payloadType === "application/super+json") {
|
||||
const newPayload = await replaceSuperJsonPayload(
|
||||
existingTaskRun.payload,
|
||||
stringifiedPayloadOverride
|
||||
);
|
||||
|
||||
return stringifyIO(newPayload);
|
||||
}
|
||||
|
||||
return conditionallyImportPacket({
|
||||
data: stringifiedPayloadOverride ?? existingTaskRun.payload,
|
||||
dataType: existingTaskRun.payloadType,
|
||||
});
|
||||
}
|
||||
|
||||
private async getExistingMetadata(existingTaskRun: TaskRun) {
|
||||
if (!existingTaskRun.seedMetadata) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return parsePacket({
|
||||
data: existingTaskRun.seedMetadata,
|
||||
dataType: existingTaskRun.seedMetadataType,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { RescheduleRunRequestBody } from "@trigger.dev/core/v3";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import { parseDelay } from "~/utils/delays";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { EnqueueDelayedRunService } from "./enqueueDelayedRun.server";
|
||||
import { engine } from "../runEngine.server";
|
||||
|
||||
export class RescheduleTaskRunService extends BaseService {
|
||||
public async call(taskRun: TaskRun, body: RescheduleRunRequestBody) {
|
||||
if (taskRun.status !== "DELAYED") {
|
||||
throw new ServiceValidationError("Cannot reschedule a run that is not delayed");
|
||||
}
|
||||
|
||||
const delay = await parseDelay(body.delay);
|
||||
|
||||
if (!delay) {
|
||||
throw new ServiceValidationError(`Invalid delay: ${body.delay}`);
|
||||
}
|
||||
|
||||
const updatedRun = await this.runStore.rescheduleRun(
|
||||
taskRun.id,
|
||||
{
|
||||
delayUntil: delay,
|
||||
queueTimestamp: delay,
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (updatedRun.engine === "V1") {
|
||||
await EnqueueDelayedRunService.reschedule(taskRun.id, delay);
|
||||
return updatedRun;
|
||||
} else {
|
||||
return engine.rescheduleDelayedRun({ runId: taskRun.id, delayUntil: delay });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
|
||||
|
||||
export class ResetIdempotencyKeyService extends BaseService {
|
||||
public async call(
|
||||
idempotencyKey: string,
|
||||
taskIdentifier: string,
|
||||
authenticatedEnv: AuthenticatedEnvironment
|
||||
): Promise<{ id: string }> {
|
||||
const { count: pgCount } = await this.runStore.clearIdempotencyKey(
|
||||
{
|
||||
byPredicate: {
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
runtimeEnvironmentId: authenticatedEnv.id,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
// Buffer-side reset: the key may belong to a buffered run that
|
||||
// hasn't materialised yet. The PG updateMany above can't see it.
|
||||
// resetIdempotency clears both the snapshot fields and the Redis
|
||||
// lookup atomically. Returns null when nothing was bound there.
|
||||
const buffer = getMollifierBuffer();
|
||||
let bufferResetFailed = false;
|
||||
const bufferResult = buffer
|
||||
? await buffer
|
||||
.resetIdempotency({
|
||||
envId: authenticatedEnv.id,
|
||||
taskIdentifier,
|
||||
idempotencyKey,
|
||||
})
|
||||
.catch((err) => {
|
||||
// Don't drop a buffer outage on the floor. We log + flag so
|
||||
// the 404 branch below can distinguish "no record anywhere"
|
||||
// (legitimate not-found) from "PG cleared nothing AND we
|
||||
// couldn't see the buffer" (partial outage — caller should
|
||||
// retry, not be told "doesn't exist").
|
||||
bufferResetFailed = true;
|
||||
logger.error("ResetIdempotencyKeyService: buffer reset failed", {
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return { clearedRunId: null };
|
||||
})
|
||||
: { clearedRunId: null };
|
||||
|
||||
const totalCount = pgCount + (bufferResult.clearedRunId ? 1 : 0);
|
||||
|
||||
if (pgCount === 0 && bufferResetFailed) {
|
||||
// PG saw nothing AND the buffer is unreachable. We can't truthfully
|
||||
// say "not found" — there may be a buffered run we can't observe.
|
||||
// Surface as 503 so the caller retries instead of being misled.
|
||||
throw new ServiceValidationError(
|
||||
"Unable to verify buffered idempotency state right now; please retry",
|
||||
503
|
||||
);
|
||||
}
|
||||
|
||||
if (totalCount === 0) {
|
||||
// PG↔buffer handoff re-check. Between the initial `pg.updateMany`
|
||||
// and the buffer reset above, a buffered run can materialise into
|
||||
// PG: the drainer's `engine.trigger` writes the row with the
|
||||
// original idempotencyKey, then `buffer.ack` clears the Redis
|
||||
// idempotency lookup (per ack's contract on
|
||||
// `packages/redis-worker/src/mollifier/buffer.ts`). Both surfaces
|
||||
// now report "nothing", but the key still lives on the freshly-
|
||||
// materialised PG row. One more conditional updateMany catches
|
||||
// that row before we 404 the customer. Cost: a single indexed
|
||||
// lookup against the writer when there's nothing to find;
|
||||
// otherwise the exact write the customer asked for (i.e., not
|
||||
// duplicative — without it the reset is silently lost).
|
||||
const { count: handoffPgCount } = await this.runStore.clearIdempotencyKey(
|
||||
{
|
||||
byPredicate: {
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
runtimeEnvironmentId: authenticatedEnv.id,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
if (handoffPgCount > 0) {
|
||||
logger.info(
|
||||
`Reset idempotency key via handoff re-check: ${idempotencyKey} for task: ${taskIdentifier} in env: ${authenticatedEnv.id}, affected ${handoffPgCount} run(s)`
|
||||
);
|
||||
return { id: idempotencyKey };
|
||||
}
|
||||
throw new ServiceValidationError(
|
||||
`No runs found with idempotency key: ${idempotencyKey} and task: ${taskIdentifier}`,
|
||||
404
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Reset idempotency key: ${idempotencyKey} for task: ${taskIdentifier} in env: ${authenticatedEnv.id}, affected ${totalCount} run(s) (pg=${pgCount}, buffered=${bufferResult.clearedRunId ? 1 : 0})`
|
||||
);
|
||||
|
||||
return { id: idempotencyKey };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { type Checkpoint } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { machinePresetFromConfig, machinePresetFromRun } from "../machinePresets.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
|
||||
import { isRestorableAttemptStatus, isRestorableRunStatus } from "../taskStatus";
|
||||
|
||||
export class RestoreCheckpointService extends BaseService {
|
||||
public async call(params: {
|
||||
eventId: string;
|
||||
isRetry?: boolean;
|
||||
}): Promise<Checkpoint | undefined> {
|
||||
logger.debug(`Restoring checkpoint`, params);
|
||||
|
||||
const checkpointEvent = await this._prisma.checkpointRestoreEvent.findFirst({
|
||||
where: {
|
||||
id: params.eventId,
|
||||
type: "CHECKPOINT",
|
||||
},
|
||||
include: {
|
||||
checkpoint: {
|
||||
include: {
|
||||
run: {
|
||||
select: {
|
||||
status: true,
|
||||
machinePreset: true,
|
||||
},
|
||||
},
|
||||
attempt: {
|
||||
select: {
|
||||
status: true,
|
||||
backgroundWorkerTask: {
|
||||
select: {
|
||||
machineConfig: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runtimeEnvironment: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!checkpointEvent) {
|
||||
logger.error("Checkpoint event not found", { eventId: params.eventId });
|
||||
return;
|
||||
}
|
||||
|
||||
const checkpoint = checkpointEvent.checkpoint;
|
||||
|
||||
if (!isRestorableRunStatus(checkpoint.run.status)) {
|
||||
logger.error("Run is unrestorable", {
|
||||
eventId: params.eventId,
|
||||
runId: checkpoint.runId,
|
||||
runStatus: checkpoint.run.status,
|
||||
attemptId: checkpoint.attemptId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isRestorableAttemptStatus(checkpoint.attempt.status) && !params.isRetry) {
|
||||
logger.error("Attempt is unrestorable", {
|
||||
eventId: params.eventId,
|
||||
runId: checkpoint.runId,
|
||||
attemptId: checkpoint.attemptId,
|
||||
attemptStatus: checkpoint.attempt.status,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const machine =
|
||||
machinePresetFromRun(checkpoint.run) ??
|
||||
machinePresetFromConfig(checkpoint.attempt.backgroundWorkerTask.machineConfig ?? {});
|
||||
|
||||
const restoreEvent = await this._prisma.checkpointRestoreEvent.findFirst({
|
||||
where: {
|
||||
checkpointId: checkpoint.id,
|
||||
type: "RESTORE",
|
||||
},
|
||||
});
|
||||
|
||||
if (restoreEvent) {
|
||||
logger.warn("Restore event already exists", {
|
||||
runId: checkpoint.runId,
|
||||
attemptId: checkpoint.attemptId,
|
||||
checkpointId: checkpoint.id,
|
||||
restoreEventId: restoreEvent.id,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const eventService = new CreateCheckpointRestoreEventService(this._prisma);
|
||||
await eventService.restore({ checkpointId: checkpoint.id });
|
||||
|
||||
socketIo.providerNamespace.emit("RESTORE", {
|
||||
version: "v1",
|
||||
type: checkpoint.type,
|
||||
location: checkpoint.location,
|
||||
reason: checkpoint.reason ?? undefined,
|
||||
imageRef: checkpoint.imageRef,
|
||||
machine,
|
||||
attemptNumber: checkpoint.attemptNumber ?? undefined,
|
||||
// identifiers
|
||||
checkpointId: checkpoint.id,
|
||||
envId: checkpoint.runtimeEnvironment.id,
|
||||
envType: checkpoint.runtimeEnvironment.type,
|
||||
orgId: checkpoint.runtimeEnvironment.organizationId,
|
||||
projectId: checkpoint.runtimeEnvironment.projectId,
|
||||
runId: checkpoint.runId,
|
||||
});
|
||||
|
||||
return checkpoint;
|
||||
}
|
||||
|
||||
async getLastCheckpointEventIfUnrestored(runId: string) {
|
||||
const event = await this._prisma.checkpointRestoreEvent.findFirst({
|
||||
where: {
|
||||
runId,
|
||||
},
|
||||
take: 1,
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
});
|
||||
|
||||
if (!event) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "CHECKPOINT") {
|
||||
return event;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
import type {
|
||||
CoordinatorToPlatformMessages,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionResult,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
|
||||
import type { Prisma, TaskRunAttempt } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
|
||||
import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
export class ResumeAttemptService extends BaseService {
|
||||
private _logger = logger;
|
||||
|
||||
public async call(
|
||||
params: InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "READY_FOR_RESUME">
|
||||
): Promise<void> {
|
||||
this._logger.debug(`ResumeAttemptService.call()`, params);
|
||||
|
||||
const latestAttemptSelect = {
|
||||
orderBy: {
|
||||
number: "desc",
|
||||
},
|
||||
take: 1,
|
||||
select: {
|
||||
id: true,
|
||||
number: true,
|
||||
status: true,
|
||||
},
|
||||
} satisfies Prisma.TaskRunInclude["attempts"];
|
||||
|
||||
const attempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: {
|
||||
friendlyId: params.attemptFriendlyId,
|
||||
},
|
||||
include: {
|
||||
taskRun: true,
|
||||
dependencies: {
|
||||
select: {
|
||||
taskRun: {
|
||||
select: {
|
||||
attempts: latestAttemptSelect,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
take: 1,
|
||||
},
|
||||
batchDependencies: {
|
||||
select: {
|
||||
items: {
|
||||
select: {
|
||||
taskRun: {
|
||||
select: {
|
||||
attempts: latestAttemptSelect,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
take: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!attempt) {
|
||||
this._logger.error("Could not find attempt", params);
|
||||
return;
|
||||
}
|
||||
|
||||
this._logger = logger.child({
|
||||
attemptId: attempt.id,
|
||||
attemptFriendlyId: attempt.friendlyId,
|
||||
taskRun: attempt.taskRun,
|
||||
});
|
||||
|
||||
if (isFinalRunStatus(attempt.taskRun.status)) {
|
||||
this._logger.error("Run is not resumable");
|
||||
return;
|
||||
}
|
||||
|
||||
let completedAttemptIds: string[] = [];
|
||||
|
||||
switch (params.type) {
|
||||
case "WAIT_FOR_DURATION": {
|
||||
this._logger.debug("Sending duration wait resume message");
|
||||
|
||||
await this.#setPostResumeStatuses(attempt);
|
||||
|
||||
socketIo.coordinatorNamespace.emit("RESUME_AFTER_DURATION", {
|
||||
version: "v1",
|
||||
attemptId: attempt.id,
|
||||
attemptFriendlyId: attempt.friendlyId,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_TASK": {
|
||||
if (attempt.dependencies.length) {
|
||||
// We only care about the latest dependency
|
||||
const dependentAttempt = attempt.dependencies[0].taskRun.attempts[0];
|
||||
|
||||
if (!dependentAttempt) {
|
||||
this._logger.error("No dependent attempt");
|
||||
return;
|
||||
}
|
||||
|
||||
completedAttemptIds = [dependentAttempt.id];
|
||||
} else {
|
||||
this._logger.error("No task dependency");
|
||||
return;
|
||||
}
|
||||
|
||||
await this.#handleDependencyResume(attempt, completedAttemptIds);
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAIT_FOR_BATCH": {
|
||||
if (attempt.batchDependencies) {
|
||||
// We only care about the latest batch dependency
|
||||
const dependentBatchItems = attempt.batchDependencies[0].items;
|
||||
|
||||
if (!dependentBatchItems) {
|
||||
this._logger.error("No dependent batch items");
|
||||
return;
|
||||
}
|
||||
|
||||
//find the best attempt for each batch item
|
||||
//it should be the most recent one in a final state
|
||||
const finalAttempts = dependentBatchItems
|
||||
.map((item) => {
|
||||
return item.taskRun.attempts
|
||||
.filter((a) => FINAL_ATTEMPT_STATUSES.includes(a.status))
|
||||
.sort((a, b) => b.number - a.number)
|
||||
.at(0);
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
completedAttemptIds = finalAttempts.map((a) => a.id);
|
||||
|
||||
if (completedAttemptIds.length !== dependentBatchItems.length) {
|
||||
this._logger.error("[ResumeAttemptService] not all batch items have attempts", {
|
||||
runId: attempt.taskRunId,
|
||||
completedAttemptIds,
|
||||
finalAttempts,
|
||||
dependentBatchItems,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
this._logger.error("No batch dependency");
|
||||
return;
|
||||
}
|
||||
|
||||
await this.#handleDependencyResume(attempt, completedAttemptIds);
|
||||
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #handleDependencyResume(attempt: TaskRunAttempt, completedAttemptIds: string[]) {
|
||||
if (completedAttemptIds.length === 0) {
|
||||
this._logger.error("No completed attempt IDs");
|
||||
return;
|
||||
}
|
||||
|
||||
const completions: TaskRunExecutionResult[] = [];
|
||||
const executions: TaskRunExecution[] = [];
|
||||
|
||||
for (const completedAttemptId of completedAttemptIds) {
|
||||
const completedAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: {
|
||||
id: completedAttemptId,
|
||||
taskRun: {
|
||||
lockedAt: {
|
||||
not: null,
|
||||
},
|
||||
lockedById: {
|
||||
not: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!completedAttempt) {
|
||||
this._logger.error("Completed attempt not found", { completedAttemptId });
|
||||
await marqs?.acknowledgeMessage(
|
||||
attempt.taskRunId,
|
||||
"Cannot find completed attempt in ResumeAttemptService"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const logger = this._logger.child({
|
||||
completedAttemptId: completedAttempt.id,
|
||||
completedAttemptFriendlyId: completedAttempt.friendlyId,
|
||||
completedRunId: completedAttempt.taskRunId,
|
||||
});
|
||||
|
||||
const resumePayload = await sharedQueueTasks.getResumePayload(completedAttempt.id);
|
||||
|
||||
if (!resumePayload) {
|
||||
logger.error("Failed to get resume payload");
|
||||
await marqs?.acknowledgeMessage(
|
||||
attempt.taskRunId,
|
||||
"Failed to get resume payload in ResumeAttemptService"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
completions.push(resumePayload.completion);
|
||||
executions.push(resumePayload.execution);
|
||||
}
|
||||
|
||||
await this.#setPostResumeStatuses(attempt);
|
||||
|
||||
socketIo.coordinatorNamespace.emit("RESUME_AFTER_DEPENDENCY", {
|
||||
version: "v1",
|
||||
runId: attempt.taskRunId,
|
||||
attemptId: attempt.id,
|
||||
attemptFriendlyId: attempt.friendlyId,
|
||||
completions,
|
||||
executions,
|
||||
});
|
||||
}
|
||||
|
||||
async #setPostResumeStatuses(attempt: TaskRunAttempt) {
|
||||
try {
|
||||
const updatedAttempt = await this._prisma.taskRunAttempt.update({
|
||||
where: {
|
||||
id: attempt.id,
|
||||
},
|
||||
data: {
|
||||
status: "EXECUTING",
|
||||
taskRun: {
|
||||
update: {
|
||||
data: {
|
||||
status: attempt.number > 1 ? "RETRYING_AFTER_FAILURE" : "EXECUTING",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
taskRun: {
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
this._logger.debug("Set post resume statuses", {
|
||||
run: {
|
||||
id: updatedAttempt.taskRun.id,
|
||||
status: updatedAttempt.taskRun.status,
|
||||
},
|
||||
attempt: {
|
||||
id: updatedAttempt.id,
|
||||
status: updatedAttempt.status,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
this._logger.error("Failed to set post resume statuses", {
|
||||
error:
|
||||
error instanceof Error
|
||||
? {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
}
|
||||
: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import type { BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
import { isV3Disabled } from "../engineDeprecation.server";
|
||||
|
||||
const finishedBatchRunStatuses = ["COMPLETED", "FAILED", "CANCELED"];
|
||||
|
||||
const BATCH_RUN_INCLUDE = {
|
||||
items: {
|
||||
select: {
|
||||
status: true,
|
||||
taskRunAttemptId: true,
|
||||
},
|
||||
},
|
||||
} satisfies Prisma.BatchTaskRunInclude;
|
||||
|
||||
type RetrieveBatchRunResult = Prisma.BatchTaskRunGetPayload<{
|
||||
include: typeof BATCH_RUN_INCLUDE;
|
||||
}>;
|
||||
|
||||
export class ResumeBatchRunService extends BaseService {
|
||||
public async call(batchRunId: string) {
|
||||
const batchRun = await this.runStore.findBatchTaskRunById(batchRunId, {
|
||||
include: BATCH_RUN_INCLUDE,
|
||||
});
|
||||
|
||||
if (!batchRun) {
|
||||
logger.error(
|
||||
"ResumeBatchRunService: Batch run doesn't exist or doesn't have a dependent attempt",
|
||||
{
|
||||
batchRunId,
|
||||
}
|
||||
);
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
// BatchTaskRun -> RuntimeEnvironment FK is dropped; resolve the env from the scalar id.
|
||||
const environment = await findEnvironmentById(batchRun.runtimeEnvironmentId);
|
||||
if (!environment) {
|
||||
logger.error("ResumeBatchRunService: Environment not found", {
|
||||
batchRunId,
|
||||
runtimeEnvironmentId: batchRun.runtimeEnvironmentId,
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
// v3 (engine V1) shutdown: don't resume batches for abandoned V1 projects. v4 is unaffected.
|
||||
// The BatchTaskRun -> RuntimeEnvironment relation is dropped, so read the engine from the
|
||||
// resolved environment's project rather than the unloaded batchRun.runtimeEnvironment relation.
|
||||
if (isV3Disabled() && environment.project.engine === "V1") {
|
||||
logger.debug("[ResumeBatchRunService] Skipping resume for shut-down v3 batch", {
|
||||
batchRunId,
|
||||
});
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
if (batchRun.batchVersion === "v3") {
|
||||
return await this.#handleV3BatchRun(batchRun, environment);
|
||||
} else {
|
||||
return await this.#handleLegacyBatchRun(batchRun, environment);
|
||||
}
|
||||
}
|
||||
|
||||
async #handleV3BatchRun(batchRun: RetrieveBatchRunResult, environment: AuthenticatedEnvironment) {
|
||||
// V3 batch runs should already be complete by the time this is called
|
||||
if (batchRun.status !== "COMPLETED") {
|
||||
logger.debug("ResumeBatchRunService: Batch run is already completed", {
|
||||
batchRunId: batchRun.id,
|
||||
batchRun: {
|
||||
id: batchRun.id,
|
||||
status: batchRun.status,
|
||||
},
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
// Even though we are in v3, we still need to check if the batch run has a dependent attempt
|
||||
if (!batchRun.dependentTaskAttemptId) {
|
||||
logger.debug("ResumeBatchRunService: Batch run doesn't have a dependent attempt", {
|
||||
batchRunId: batchRun.id,
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
return await this.#handleDependentTaskAttempt(
|
||||
batchRun,
|
||||
batchRun.dependentTaskAttemptId,
|
||||
environment
|
||||
);
|
||||
}
|
||||
|
||||
async #handleLegacyBatchRun(
|
||||
batchRun: RetrieveBatchRunResult,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
if (batchRun.status === "COMPLETED") {
|
||||
logger.debug("ResumeBatchRunService: Batch run is already completed", {
|
||||
batchRunId: batchRun.id,
|
||||
batchRun: {
|
||||
id: batchRun.id,
|
||||
status: batchRun.status,
|
||||
},
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
if (batchRun.batchVersion === "v2") {
|
||||
if (batchRun.items.length < batchRun.runCount) {
|
||||
logger.debug("ResumeBatchRunService: All items aren't yet completed [v2]", {
|
||||
batchRunId: batchRun.id,
|
||||
batchRun: {
|
||||
id: batchRun.id,
|
||||
status: batchRun.status,
|
||||
itemsLength: batchRun.items.length,
|
||||
runCount: batchRun.runCount,
|
||||
},
|
||||
});
|
||||
|
||||
return "PENDING";
|
||||
}
|
||||
}
|
||||
|
||||
if (batchRun.items.some((item) => !finishedBatchRunStatuses.includes(item.status))) {
|
||||
logger.debug("ResumeBatchRunService: All items aren't yet completed [v1]", {
|
||||
batchRunId: batchRun.id,
|
||||
batchRun: {
|
||||
id: batchRun.id,
|
||||
status: batchRun.status,
|
||||
},
|
||||
});
|
||||
|
||||
return "PENDING";
|
||||
}
|
||||
|
||||
// If we are in development, or there is no dependent attempt, we can just mark the batch as completed and return
|
||||
if (environment.type === "DEVELOPMENT" || !batchRun.dependentTaskAttemptId) {
|
||||
// We need to update the batchRun status so we don't resume it again
|
||||
await this.runStore.updateBatchTaskRun({
|
||||
where: {
|
||||
id: batchRun.id,
|
||||
},
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
return await this.#handleDependentTaskAttempt(
|
||||
batchRun,
|
||||
batchRun.dependentTaskAttemptId,
|
||||
environment
|
||||
);
|
||||
}
|
||||
|
||||
async #handleDependentTaskAttempt(
|
||||
batchRun: RetrieveBatchRunResult,
|
||||
dependentTaskAttemptId: string,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
const dependentTaskAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
where: {
|
||||
id: dependentTaskAttemptId,
|
||||
},
|
||||
select: {
|
||||
status: true,
|
||||
id: true,
|
||||
taskRun: {
|
||||
select: {
|
||||
id: true,
|
||||
queue: true,
|
||||
taskIdentifier: true,
|
||||
concurrencyKey: true,
|
||||
createdAt: true,
|
||||
queueTimestamp: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!dependentTaskAttempt) {
|
||||
logger.error("ResumeBatchRunService: Dependent attempt not found", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttemptId: batchRun.dependentTaskAttemptId,
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
// This batch has a dependent attempt and just finalized, we should resume that attempt
|
||||
const dependentRun = dependentTaskAttempt.taskRun;
|
||||
|
||||
if (dependentTaskAttempt.status === "PAUSED" && batchRun.checkpointEventId) {
|
||||
logger.debug("ResumeBatchRunService: Attempt is paused and has a checkpoint event", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
// We need to update the batchRun status so we don't resume it again
|
||||
const wasUpdated = await this.#setBatchToResumedOnce(batchRun);
|
||||
|
||||
if (wasUpdated) {
|
||||
logger.debug("ResumeBatchRunService: Resuming dependent run with checkpoint", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttemptId: dependentTaskAttempt.id,
|
||||
});
|
||||
|
||||
await marqs.enqueueMessage(
|
||||
environment,
|
||||
dependentRun.queue,
|
||||
dependentRun.id,
|
||||
{
|
||||
type: "RESUME",
|
||||
completedAttemptIds: [],
|
||||
resumableAttemptId: dependentTaskAttempt.id,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
taskIdentifier: dependentTaskAttempt.taskRun.taskIdentifier,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
},
|
||||
dependentRun.concurrencyKey ?? undefined,
|
||||
dependentRun.queueTimestamp ?? dependentRun.createdAt,
|
||||
undefined,
|
||||
"resume"
|
||||
);
|
||||
|
||||
return "COMPLETED";
|
||||
} else {
|
||||
logger.debug("ResumeBatchRunService: with checkpoint was already completed", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
hasCheckpointEvent: !!batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
return "ALREADY_COMPLETED";
|
||||
}
|
||||
} else {
|
||||
logger.debug("ResumeBatchRunService: attempt is not paused or there's no checkpoint event", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
hasCheckpointEvent: !!batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
if (dependentTaskAttempt.status === "PAUSED" && !batchRun.checkpointEventId) {
|
||||
// In case of race conditions the status can be PAUSED without a checkpoint event
|
||||
// When the checkpoint is created, it will continue the run
|
||||
logger.error("ResumeBatchRunService: attempt is paused but there's no checkpoint event", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
hasCheckpointEvent: !!batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
return "ERROR";
|
||||
}
|
||||
|
||||
// We need to update the batchRun status so we don't resume it again
|
||||
const wasUpdated = await this.#setBatchToResumedOnce(batchRun);
|
||||
|
||||
if (wasUpdated) {
|
||||
logger.debug("ResumeBatchRunService: Resuming dependent run without checkpoint", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
hasCheckpointEvent: !!batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
await marqs.requeueMessage(
|
||||
dependentRun.id,
|
||||
{
|
||||
type: "RESUME",
|
||||
completedAttemptIds: batchRun.items
|
||||
.map((item) => item.taskRunAttemptId)
|
||||
.filter(Boolean),
|
||||
resumableAttemptId: dependentTaskAttempt.id,
|
||||
checkpointEventId: batchRun.checkpointEventId ?? undefined,
|
||||
taskIdentifier: dependentTaskAttempt.taskRun.taskIdentifier,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
},
|
||||
(
|
||||
dependentTaskAttempt.taskRun.queueTimestamp ?? dependentTaskAttempt.taskRun.createdAt
|
||||
).getTime(),
|
||||
"resume"
|
||||
);
|
||||
|
||||
return "COMPLETED";
|
||||
} else {
|
||||
logger.debug("ResumeBatchRunService: without checkpoint was already completed", {
|
||||
batchRunId: batchRun.id,
|
||||
dependentTaskAttempt: dependentTaskAttempt,
|
||||
checkpointEventId: batchRun.checkpointEventId,
|
||||
hasCheckpointEvent: !!batchRun.checkpointEventId,
|
||||
});
|
||||
|
||||
return "ALREADY_COMPLETED";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #setBatchToResumedOnce(batchRun: BatchTaskRun) {
|
||||
// v3 batches don't use the status for deciding whether a batch has been resumed
|
||||
if (batchRun.batchVersion === "v3") {
|
||||
const result = await this.runStore.updateManyBatchTaskRun({
|
||||
where: {
|
||||
id: batchRun.id,
|
||||
resumedAt: null,
|
||||
},
|
||||
data: {
|
||||
resumedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
if (result.count > 0) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const result = await this.runStore.updateManyBatchTaskRun({
|
||||
where: {
|
||||
id: batchRun.id,
|
||||
status: {
|
||||
not: "COMPLETED", // Ensure the status is not already "COMPLETED"
|
||||
},
|
||||
},
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
},
|
||||
});
|
||||
|
||||
if (result.count > 0) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(
|
||||
batchRunId: string,
|
||||
skipJobKey: boolean,
|
||||
tx?: PrismaClientOrTransaction,
|
||||
runAt?: Date
|
||||
) {
|
||||
if (tx) {
|
||||
logger.debug("ResumeBatchRunService: Enqueuing resume batch run using workerQueue", {
|
||||
batchRunId,
|
||||
skipJobKey,
|
||||
runAt,
|
||||
});
|
||||
|
||||
return await workerQueue.enqueue(
|
||||
"v3.resumeBatchRun",
|
||||
{
|
||||
batchRunId,
|
||||
},
|
||||
{
|
||||
jobKey: skipJobKey ? undefined : `resumeBatchRun-${batchRunId}`,
|
||||
runAt,
|
||||
tx,
|
||||
}
|
||||
);
|
||||
} else {
|
||||
logger.debug("ResumeBatchRunService: Enqueuing resume batch run using commonWorker", {
|
||||
batchRunId,
|
||||
skipJobKey,
|
||||
runAt,
|
||||
});
|
||||
|
||||
return await commonWorker.enqueue({
|
||||
id: skipJobKey ? undefined : `resumeBatchRun-${batchRunId}`,
|
||||
job: "v3.resumeBatchRun",
|
||||
payload: {
|
||||
batchRunId,
|
||||
},
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
import type { Prisma } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { ResumeBatchRunService } from "./resumeBatchRun.server";
|
||||
import { ResumeTaskDependencyService } from "./resumeTaskDependency.server";
|
||||
import { $transaction } from "~/db.server";
|
||||
import { completeBatchTaskRunItemV3 } from "./batchTriggerV3.server";
|
||||
|
||||
type Output =
|
||||
| {
|
||||
success: true;
|
||||
action:
|
||||
| "resume-scheduled"
|
||||
| "batch-resume-scheduled"
|
||||
| "no-dependencies"
|
||||
| "not-finished"
|
||||
| "dev";
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
const taskRunDependencySelect = {
|
||||
select: {
|
||||
id: true,
|
||||
taskRunId: true,
|
||||
taskRun: {
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
runtimeEnvironment: {
|
||||
select: {
|
||||
type: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dependentAttempt: {
|
||||
select: {
|
||||
id: true,
|
||||
},
|
||||
},
|
||||
dependentBatchRun: {
|
||||
select: {
|
||||
id: true,
|
||||
batchVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
type Dependency = Prisma.TaskRunDependencyGetPayload<typeof taskRunDependencySelect>;
|
||||
|
||||
/** This will resume a dependent (parent) run if there is one and it makes sense. */
|
||||
export class ResumeDependentParentsService extends BaseService {
|
||||
public async call({ id }: { id: string }): Promise<Output> {
|
||||
try {
|
||||
const dependency = await this._prisma.taskRunDependency.findFirst({
|
||||
...taskRunDependencySelect,
|
||||
where: {
|
||||
taskRunId: id,
|
||||
},
|
||||
});
|
||||
|
||||
logger.log("ResumeDependentParentsService: tried to find dependency", {
|
||||
runId: id,
|
||||
dependency: dependency,
|
||||
});
|
||||
|
||||
if (!dependency) {
|
||||
logger.log("ResumeDependentParentsService: dependency not found", {
|
||||
runId: id,
|
||||
});
|
||||
|
||||
//no dependency, that's fine most runs won't have one.
|
||||
return {
|
||||
success: true,
|
||||
action: "no-dependencies",
|
||||
};
|
||||
}
|
||||
|
||||
if (dependency.taskRun.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
return {
|
||||
success: true,
|
||||
action: "dev",
|
||||
};
|
||||
}
|
||||
|
||||
if (!isFinalRunStatus(dependency.taskRun.status)) {
|
||||
logger.debug(
|
||||
"ResumeDependentParentsService: run not finished yet, can't resume parent yet",
|
||||
{
|
||||
runId: id,
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
// the child run isn't finished yet, so we can't resume the parent yet.
|
||||
return {
|
||||
success: true,
|
||||
action: "not-finished",
|
||||
};
|
||||
}
|
||||
|
||||
if (dependency.dependentAttempt) {
|
||||
return this.#singleRunDependency(dependency);
|
||||
} else if (dependency.dependentBatchRun) {
|
||||
return this.#batchRunDependency(dependency);
|
||||
} else {
|
||||
logger.error("ResumeDependentParentsService: dependency has no dependencies", {
|
||||
runId: id,
|
||||
dependency,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Dependency has no dependencies (single or batch)`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : JSON.stringify(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async #singleRunDependency(dependency: Dependency): Promise<Output> {
|
||||
logger.debug(
|
||||
`ResumeDependentParentsService.singleRunDependency(): Resuming dependent parent for run`,
|
||||
{
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
const lastAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
where: {
|
||||
taskRunId: dependency.taskRunId,
|
||||
},
|
||||
orderBy: {
|
||||
id: "desc",
|
||||
},
|
||||
});
|
||||
|
||||
if (!lastAttempt) {
|
||||
logger.error(
|
||||
"ResumeDependentParentsService.singleRunDependency(): dependency child attempt not found",
|
||||
{
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Dependency child attempt not found for run ${dependency.taskRunId}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!isFinalAttemptStatus(lastAttempt.status)) {
|
||||
//We still want to continue if this happens because the run is final but log it
|
||||
logger.error(
|
||||
"ResumeDependentParentsService.singleRunDependency(): dependency child attempt not final, but the run is.",
|
||||
{
|
||||
dependency,
|
||||
lastAttempt,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Dependency child attempt not final, but the run is`,
|
||||
};
|
||||
}
|
||||
|
||||
//resume the dependent task
|
||||
await ResumeTaskDependencyService.enqueue(dependency.id, lastAttempt.id);
|
||||
return {
|
||||
success: true,
|
||||
action: "resume-scheduled",
|
||||
};
|
||||
}
|
||||
|
||||
async #batchRunDependency(dependency: Dependency): Promise<Output> {
|
||||
logger.debug(
|
||||
`ResumeDependentParentsService.batchRunDependency(): Resuming dependent batch for run`,
|
||||
{
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
if (!dependency.dependentBatchRun) {
|
||||
logger.error(
|
||||
"ResumeDependentParentsService.batchRunDependency(): dependency has no dependent batch",
|
||||
{
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Dependency has no dependent batch`,
|
||||
};
|
||||
}
|
||||
|
||||
const lastAttempt = await this._prisma.taskRunAttempt.findFirst({
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
where: {
|
||||
taskRunId: dependency.taskRunId,
|
||||
},
|
||||
orderBy: {
|
||||
id: "desc",
|
||||
},
|
||||
});
|
||||
|
||||
if (!lastAttempt) {
|
||||
logger.error(
|
||||
"ResumeDependentParentsService.singleRunDependency(): dependency child attempt not found",
|
||||
{
|
||||
dependency,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `Dependency child attempt not found for run ${dependency.taskRunId}`,
|
||||
};
|
||||
}
|
||||
|
||||
logger.log(
|
||||
"ResumeDependentParentsService.batchRunDependency(): Setting the batchTaskRunItem to COMPLETED",
|
||||
{
|
||||
dependency,
|
||||
lastAttempt,
|
||||
}
|
||||
);
|
||||
|
||||
if (dependency.dependentBatchRun!.batchVersion === "v3") {
|
||||
const batchTaskRunItem = await this._prisma.batchTaskRunItem.findFirst({
|
||||
where: {
|
||||
batchTaskRunId: dependency.dependentBatchRun!.id,
|
||||
taskRunId: dependency.taskRunId,
|
||||
},
|
||||
});
|
||||
|
||||
if (batchTaskRunItem) {
|
||||
await completeBatchTaskRunItemV3(
|
||||
batchTaskRunItem.id,
|
||||
batchTaskRunItem.batchTaskRunId,
|
||||
this._prisma,
|
||||
true,
|
||||
lastAttempt.id
|
||||
);
|
||||
} else {
|
||||
logger.debug(
|
||||
"ResumeDependentParentsService.batchRunDependency() v3: batchTaskRunItem not found",
|
||||
{
|
||||
dependency,
|
||||
lastAttempt,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else {
|
||||
await $transaction(this._prisma, async (tx) => {
|
||||
await tx.batchTaskRunItem.update({
|
||||
where: {
|
||||
batchTaskRunId_taskRunId: {
|
||||
batchTaskRunId: dependency.dependentBatchRun!.id,
|
||||
taskRunId: dependency.taskRunId,
|
||||
},
|
||||
},
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
taskRunAttemptId: lastAttempt.id,
|
||||
},
|
||||
});
|
||||
|
||||
await ResumeBatchRunService.enqueue(dependency.dependentBatchRun!.id, false, tx);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
action: "batch-resume-scheduled",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
import type { TaskRunDependency } from "@trigger.dev/database";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { isV3Disabled } from "../engineDeprecation.server";
|
||||
|
||||
export class ResumeTaskDependencyService extends BaseService {
|
||||
public async call(dependencyId: string, sourceTaskAttemptId: string) {
|
||||
const dependency = await this._prisma.taskRunDependency.findFirst({
|
||||
where: { id: dependencyId },
|
||||
include: {
|
||||
taskRun: {
|
||||
include: {
|
||||
runtimeEnvironment: {
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dependentAttempt: {
|
||||
include: {
|
||||
taskRun: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Dependencies with a dependentBatchRun are handled already by the ResumeBatchRunService
|
||||
if (!dependency || !dependency.dependentAttempt) {
|
||||
return;
|
||||
}
|
||||
|
||||
// v3 (engine V1) shutdown: don't resume dependencies for abandoned V1 runs. v4 is unaffected.
|
||||
if (isV3Disabled() && dependency.taskRun.engine === "V1") {
|
||||
logger.debug("[ResumeTaskDependencyService] Skipping resume for shut-down v3 run", {
|
||||
dependencyId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (dependency.taskRun.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
return;
|
||||
}
|
||||
|
||||
const dependentRun = dependency.dependentAttempt.taskRun;
|
||||
|
||||
if (dependency.dependentAttempt.status === "PAUSED" && dependency.checkpointEventId) {
|
||||
logger.debug(
|
||||
"Task dependency resume: Attempt is paused and there's a checkpoint. Enqueuing resume with checkpoint.",
|
||||
{
|
||||
attemptId: dependency.id,
|
||||
dependentAttempt: dependency.dependentAttempt,
|
||||
checkpointEventId: dependency.checkpointEventId,
|
||||
hasCheckpointEvent: !!dependency.checkpointEventId,
|
||||
runId: dependentRun.id,
|
||||
}
|
||||
);
|
||||
|
||||
const wasUpdated = await this.#setDependencyToResumedOnce(dependency);
|
||||
|
||||
if (!wasUpdated) {
|
||||
logger.debug("Task dependency resume: Attempt with checkpoint was already resumed", {
|
||||
attemptId: dependency.id,
|
||||
dependentAttempt: dependency.dependentAttempt,
|
||||
checkpointEventId: dependency.checkpointEventId,
|
||||
hasCheckpointEvent: !!dependency.checkpointEventId,
|
||||
runId: dependentRun.id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: use the new priority queue thingie
|
||||
await marqs?.enqueueMessage(
|
||||
dependency.taskRun.runtimeEnvironment,
|
||||
dependentRun.queue,
|
||||
dependentRun.id,
|
||||
{
|
||||
type: "RESUME",
|
||||
completedAttemptIds: [sourceTaskAttemptId],
|
||||
resumableAttemptId: dependency.dependentAttempt.id,
|
||||
checkpointEventId: dependency.checkpointEventId,
|
||||
taskIdentifier: dependency.taskRun.taskIdentifier,
|
||||
projectId: dependency.taskRun.runtimeEnvironment.projectId,
|
||||
environmentId: dependency.taskRun.runtimeEnvironment.id,
|
||||
environmentType: dependency.taskRun.runtimeEnvironment.type,
|
||||
},
|
||||
dependentRun.concurrencyKey ?? undefined,
|
||||
dependentRun.queueTimestamp ?? dependentRun.createdAt,
|
||||
undefined,
|
||||
"resume"
|
||||
);
|
||||
} else {
|
||||
logger.debug("Task dependency resume: Attempt is not paused or there's no checkpoint event", {
|
||||
attemptId: dependency.id,
|
||||
dependentAttempt: dependency.dependentAttempt,
|
||||
checkpointEventId: dependency.checkpointEventId,
|
||||
hasCheckpointEvent: !!dependency.checkpointEventId,
|
||||
runId: dependentRun.id,
|
||||
});
|
||||
|
||||
if (dependency.dependentAttempt.status === "PAUSED" && !dependency.checkpointEventId) {
|
||||
// In case of race conditions the status can be PAUSED without a checkpoint event
|
||||
// When the checkpoint is created, it will continue the run
|
||||
logger.error("Task dependency resume: Attempt is paused but there's no checkpoint event", {
|
||||
attemptId: dependency.id,
|
||||
dependentAttemptId: dependency.dependentAttempt.id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const wasUpdated = await this.#setDependencyToResumedOnce(dependency);
|
||||
|
||||
if (!wasUpdated) {
|
||||
logger.debug("Task dependency resume: Attempt without checkpoint was already resumed", {
|
||||
attemptId: dependency.id,
|
||||
dependentAttempt: dependency.dependentAttempt,
|
||||
checkpointEventId: dependency.checkpointEventId,
|
||||
hasCheckpointEvent: !!dependency.checkpointEventId,
|
||||
runId: dependentRun.id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await marqs.requeueMessage(
|
||||
dependentRun.id,
|
||||
{
|
||||
type: "RESUME",
|
||||
completedAttemptIds: [sourceTaskAttemptId],
|
||||
resumableAttemptId: dependency.dependentAttempt.id,
|
||||
checkpointEventId: dependency.checkpointEventId ?? undefined,
|
||||
taskIdentifier: dependency.taskRun.taskIdentifier,
|
||||
projectId: dependency.taskRun.runtimeEnvironment.projectId,
|
||||
environmentId: dependency.taskRun.runtimeEnvironment.id,
|
||||
environmentType: dependency.taskRun.runtimeEnvironment.type,
|
||||
},
|
||||
(dependentRun.queueTimestamp ?? dependentRun.createdAt).getTime(),
|
||||
"resume"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #setDependencyToResumedOnce(dependency: TaskRunDependency) {
|
||||
const result = await this._prisma.taskRunDependency.updateMany({
|
||||
where: {
|
||||
id: dependency.id,
|
||||
resumedAt: null,
|
||||
},
|
||||
data: {
|
||||
resumedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
// Check if any records were updated
|
||||
if (result.count > 0) {
|
||||
// The status was changed, so we return true
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(dependencyId: string, sourceTaskAttemptId: string, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
job: "v3.resumeTaskDependency",
|
||||
payload: {
|
||||
dependencyId,
|
||||
sourceTaskAttemptId,
|
||||
},
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { commonWorker } from "../commonWorker.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { isV3Disabled } from "../engineDeprecation.server";
|
||||
|
||||
export class RetryAttemptService extends BaseService {
|
||||
public async call(runId: string) {
|
||||
const taskRun = await this.runStore.findRun({ id: runId }, this._prisma);
|
||||
|
||||
if (!taskRun) {
|
||||
logger.error("Task run not found", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
// v3 (engine V1) shutdown: don't retry abandoned V1 runs. v4 is unaffected.
|
||||
if (isV3Disabled() && taskRun.engine === "V1") {
|
||||
logger.debug("[RetryAttemptService] Skipping retry for shut-down v3 run", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
socketIo.coordinatorNamespace.emit("READY_FOR_RETRY", {
|
||||
version: "v1",
|
||||
runId,
|
||||
});
|
||||
}
|
||||
|
||||
static async enqueue(runId: string, runAt?: Date) {
|
||||
return await commonWorker.enqueue({
|
||||
id: `retryAttempt:${runId}`,
|
||||
job: "v3.retryAttempt",
|
||||
payload: {
|
||||
runId,
|
||||
},
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
type Options = {
|
||||
projectId: string;
|
||||
userId: string;
|
||||
friendlyId: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
export class SetActiveOnTaskScheduleService extends BaseService {
|
||||
public async call({ projectId, userId, friendlyId, active }: Options) {
|
||||
//first check that the user has access to the project
|
||||
const project = await this._prisma.project.findFirst({
|
||||
where: {
|
||||
id: projectId,
|
||||
organization: {
|
||||
members: {
|
||||
some: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!project) {
|
||||
throw new Error("User does not have access to the project");
|
||||
}
|
||||
|
||||
try {
|
||||
const schedule = await this._prisma.taskSchedule.findFirst({
|
||||
where: {
|
||||
friendlyId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!schedule) {
|
||||
throw new Error("Schedule not found");
|
||||
}
|
||||
|
||||
if (schedule.type === "DECLARATIVE") {
|
||||
throw new Error("Cannot enable/disable declarative schedules");
|
||||
}
|
||||
|
||||
await this._prisma.taskSchedule.update({
|
||||
where: {
|
||||
friendlyId,
|
||||
},
|
||||
data: {
|
||||
active,
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Error ${active ? "enabling" : "disabling"} schedule: ${
|
||||
e instanceof Error ? e.message : JSON.stringify(e)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import { setBranchesAddOn } from "~/services/platform.v3.server";
|
||||
import assertNever from "assert-never";
|
||||
import { sendToPlain } from "~/utils/plain.server";
|
||||
import { uiComponent } from "@team-plain/typescript-sdk";
|
||||
|
||||
type Input = {
|
||||
userId: string;
|
||||
organizationId: string;
|
||||
action: "purchase" | "quota-increase";
|
||||
amount: number;
|
||||
};
|
||||
|
||||
type Result =
|
||||
| {
|
||||
success: true;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export class SetBranchesAddOnService extends BaseService {
|
||||
async call({ userId, organizationId, action, amount }: Input): Promise<Result> {
|
||||
switch (action) {
|
||||
case "purchase": {
|
||||
const result = await setBranchesAddOn(organizationId, amount);
|
||||
if (!result) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Failed to update preview branches",
|
||||
};
|
||||
}
|
||||
|
||||
switch (result.result) {
|
||||
case "success": {
|
||||
return { success: true };
|
||||
}
|
||||
case "error": {
|
||||
return { success: false, error: result.error };
|
||||
}
|
||||
case "max_quota_reached": {
|
||||
return {
|
||||
success: false,
|
||||
error: `You can't purchase more than ${result.maxQuota} preview branches without requesting an increase.`,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return {
|
||||
success: false,
|
||||
error: "Failed to update preview branches, unknown result.",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
case "quota-increase": {
|
||||
const user = await this._replica.user.findFirst({
|
||||
where: { id: userId },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
return { success: false, error: "No matching user found." };
|
||||
}
|
||||
|
||||
const organization = await this._replica.organization.findFirst({
|
||||
select: { title: true },
|
||||
where: { id: organizationId },
|
||||
});
|
||||
|
||||
const [error] = await tryCatch(
|
||||
sendToPlain({
|
||||
userId,
|
||||
email: user.email,
|
||||
name: user.name ?? user.displayName ?? user.email,
|
||||
title: `Preview branches quota request: ${amount}`,
|
||||
components: [
|
||||
uiComponent.text({
|
||||
text: `Org: ${organization?.title} (${organizationId})`,
|
||||
}),
|
||||
uiComponent.divider({ spacingSize: "M" }),
|
||||
uiComponent.text({
|
||||
text: `Total preview branches requested: ${amount}`,
|
||||
}),
|
||||
],
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
default: {
|
||||
assertNever(action);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { ManageConcurrencyPresenter } from "~/presenters/v3/ManageConcurrencyPresenter.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { setConcurrencyAddOn } from "~/services/platform.v3.server";
|
||||
import assertNever from "assert-never";
|
||||
import { sendToPlain } from "~/utils/plain.server";
|
||||
import { uiComponent } from "@team-plain/typescript-sdk";
|
||||
|
||||
type Input = {
|
||||
userId: string;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
action: "purchase" | "quota-increase";
|
||||
amount: number;
|
||||
};
|
||||
|
||||
type Result =
|
||||
| {
|
||||
success: true;
|
||||
}
|
||||
| {
|
||||
success: false;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export class SetConcurrencyAddOnService extends BaseService {
|
||||
async call({ userId, projectId, organizationId, action, amount }: Input): Promise<Result> {
|
||||
// fetch the current concurrency
|
||||
const presenter = new ManageConcurrencyPresenter(this._prisma, this._replica);
|
||||
const [error, result] = await tryCatch(
|
||||
presenter.call({
|
||||
userId,
|
||||
projectId,
|
||||
organizationId,
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Unknown error",
|
||||
};
|
||||
}
|
||||
|
||||
const currentConcurrency = result.extraConcurrency;
|
||||
const totalExtraConcurrency = amount;
|
||||
|
||||
switch (action) {
|
||||
case "purchase": {
|
||||
const updatedConcurrency = await setConcurrencyAddOn(organizationId, totalExtraConcurrency);
|
||||
if (!updatedConcurrency) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Failed to update concurrency",
|
||||
};
|
||||
}
|
||||
|
||||
switch (updatedConcurrency?.result) {
|
||||
case "success": {
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
case "error": {
|
||||
return {
|
||||
success: false,
|
||||
error: updatedConcurrency.error,
|
||||
};
|
||||
}
|
||||
case "max_quota_reached": {
|
||||
return {
|
||||
success: false,
|
||||
error: `You can't purchase more than ${updatedConcurrency.maxQuota} concurrency without requesting an increase.`,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
return {
|
||||
success: false,
|
||||
error: "Failed to update concurrency, unknown result.",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
case "quota-increase": {
|
||||
const user = await this._replica.user.findFirst({
|
||||
where: { id: userId },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
return {
|
||||
success: false,
|
||||
error: "No matching user found.",
|
||||
};
|
||||
}
|
||||
|
||||
const organization = await this._replica.organization.findFirst({
|
||||
select: {
|
||||
title: true,
|
||||
},
|
||||
where: { id: organizationId },
|
||||
});
|
||||
|
||||
const [error, _result] = await tryCatch(
|
||||
sendToPlain({
|
||||
userId,
|
||||
email: user.email,
|
||||
name: user.name ?? user.displayName ?? user.email,
|
||||
title: `Concurrency quota request: ${totalExtraConcurrency}`,
|
||||
components: [
|
||||
uiComponent.text({
|
||||
text: `Org: ${organization?.title} (${organizationId})`,
|
||||
}),
|
||||
uiComponent.divider({ spacingSize: "M" }),
|
||||
uiComponent.text({
|
||||
text: `Total concurrency (set this): ${totalExtraConcurrency}`,
|
||||
}),
|
||||
uiComponent.text({
|
||||
text: `Current extra concurrency: ${currentConcurrency}`,
|
||||
}),
|
||||
uiComponent.text({
|
||||
text: `Amount requested: ${amount}`,
|
||||
}),
|
||||
],
|
||||
})
|
||||
);
|
||||
|
||||
if (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
assertNever(action);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user