chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,150 @@
|
||||
import {
|
||||
type ProjectAlertChannel,
|
||||
type ProjectAlertType,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import { nanoid } from "nanoid";
|
||||
import { env } from "~/env.server";
|
||||
import { findProjectByRef } from "~/models/project.server";
|
||||
import { encryptSecret } from "~/services/secrets/secretStore.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
|
||||
import { BaseService, ServiceValidationError } from "../baseService.server";
|
||||
import { assertSafeWebhookUrl, UnsafeWebhookUrlError } from "./safeWebhookUrl.server";
|
||||
|
||||
export type CreateAlertChannelOptions = {
|
||||
name: string;
|
||||
alertTypes: ProjectAlertType[];
|
||||
environmentTypes: RuntimeEnvironmentType[];
|
||||
deduplicationKey?: string;
|
||||
channel:
|
||||
| {
|
||||
type: "EMAIL";
|
||||
email: string;
|
||||
}
|
||||
| {
|
||||
type: "WEBHOOK";
|
||||
url: string;
|
||||
secret?: string;
|
||||
}
|
||||
| {
|
||||
type: "SLACK";
|
||||
channelId: string;
|
||||
channelName: string;
|
||||
integrationId: string | undefined;
|
||||
};
|
||||
};
|
||||
|
||||
export class CreateAlertChannelService extends BaseService {
|
||||
public async call(
|
||||
projectRef: string,
|
||||
userId: string,
|
||||
options: CreateAlertChannelOptions
|
||||
): Promise<ProjectAlertChannel> {
|
||||
const project = await findProjectByRef(projectRef, userId);
|
||||
|
||||
if (!project) {
|
||||
throw new ServiceValidationError("Project not found");
|
||||
}
|
||||
|
||||
// Validate webhook URLs here (not per-route) so every caller is covered.
|
||||
// Delivery re-validates at connect time via safeWebhookFetch.
|
||||
if (options.channel.type === "WEBHOOK") {
|
||||
try {
|
||||
await assertSafeWebhookUrl(options.channel.url);
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafeWebhookUrlError) {
|
||||
throw new ServiceValidationError(error.message);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const environmentTypes =
|
||||
options.environmentTypes.length === 0
|
||||
? (["STAGING", "PRODUCTION"] satisfies RuntimeEnvironmentType[])
|
||||
: options.environmentTypes;
|
||||
|
||||
const existingAlertChannel = options.deduplicationKey
|
||||
? await this._prisma.projectAlertChannel.findFirst({
|
||||
where: {
|
||||
projectId: project.id,
|
||||
deduplicationKey: options.deduplicationKey,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
if (existingAlertChannel) {
|
||||
const updated = await this._prisma.projectAlertChannel.update({
|
||||
where: { id: existingAlertChannel.id },
|
||||
data: {
|
||||
name: options.name,
|
||||
alertTypes: options.alertTypes,
|
||||
type: options.channel.type,
|
||||
properties: await this.#createProperties(options.channel),
|
||||
environmentTypes,
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.alertTypes.includes("ERROR_GROUP")) {
|
||||
await this.#scheduleErrorAlertEvaluation(project.id);
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
const alertChannel = await this._prisma.projectAlertChannel.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("alert_channel"),
|
||||
name: options.name,
|
||||
alertTypes: options.alertTypes,
|
||||
projectId: project.id,
|
||||
type: options.channel.type,
|
||||
properties: await this.#createProperties(options.channel),
|
||||
enabled: true,
|
||||
deduplicationKey: options.deduplicationKey,
|
||||
userProvidedDeduplicationKey: options.deduplicationKey ? true : false,
|
||||
environmentTypes,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.alertTypes.includes("ERROR_GROUP")) {
|
||||
await this.#scheduleErrorAlertEvaluation(project.id);
|
||||
}
|
||||
|
||||
return alertChannel;
|
||||
}
|
||||
|
||||
async #scheduleErrorAlertEvaluation(projectId: string): Promise<void> {
|
||||
await alertsWorker.enqueue({
|
||||
id: `evaluateErrorAlerts:${projectId}`,
|
||||
job: "v3.evaluateErrorAlerts",
|
||||
payload: {
|
||||
projectId,
|
||||
scheduledAt: Date.now(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async #createProperties(channel: CreateAlertChannelOptions["channel"]) {
|
||||
switch (channel.type) {
|
||||
case "EMAIL":
|
||||
return {
|
||||
email: channel.email,
|
||||
};
|
||||
case "WEBHOOK":
|
||||
return {
|
||||
url: channel.url,
|
||||
secret: await encryptSecret(env.ENCRYPTION_KEY, channel.secret ?? nanoid()),
|
||||
version: "v2",
|
||||
};
|
||||
case "SLACK":
|
||||
return {
|
||||
channelId: channel.channelId,
|
||||
channelName: channel.channelName,
|
||||
integrationId: channel.integrationId,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,423 @@
|
||||
import {
|
||||
type ChatPostMessageArguments,
|
||||
ErrorCode,
|
||||
type WebAPIPlatformError,
|
||||
type WebAPIRateLimitedError,
|
||||
} from "@slack/web-api";
|
||||
import { type ProjectAlertChannelType } from "@trigger.dev/database";
|
||||
import assertNever from "assert-never";
|
||||
import { prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { v3ErrorPath } from "~/utils/pathBuilder";
|
||||
import {
|
||||
isIntegrationForService,
|
||||
type OrganizationIntegrationForService,
|
||||
OrgIntegrationRepository,
|
||||
} from "~/models/orgIntegration.server";
|
||||
import {
|
||||
ProjectAlertEmailProperties,
|
||||
ProjectAlertSlackProperties,
|
||||
ProjectAlertWebhookProperties,
|
||||
} from "~/models/projectAlert.server";
|
||||
import { sendAlertEmail } from "~/services/email.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { decryptSecret } from "~/services/secrets/secretStore.server";
|
||||
import { subtle } from "crypto";
|
||||
import { generateErrorGroupWebhookPayload } from "./errorGroupWebhook.server";
|
||||
import { safeWebhookFetch } from "./safeWebhookFetch.server";
|
||||
|
||||
type ErrorAlertClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
interface ErrorAlertPayload {
|
||||
channelId: string;
|
||||
projectId: string;
|
||||
classification: ErrorAlertClassification;
|
||||
error: {
|
||||
fingerprint: string;
|
||||
environmentId: string;
|
||||
environmentSlug: string;
|
||||
environmentName: string;
|
||||
taskIdentifier: string;
|
||||
errorType: string;
|
||||
errorMessage: string;
|
||||
sampleStackTrace: string;
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
occurrenceCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
class SkipRetryError extends Error {}
|
||||
|
||||
export class DeliverErrorGroupAlertService {
|
||||
async call(payload: ErrorAlertPayload): Promise<void> {
|
||||
const channel = await prisma.projectAlertChannel.findFirst({
|
||||
where: { id: payload.channelId, enabled: true },
|
||||
include: {
|
||||
project: {
|
||||
include: {
|
||||
organization: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!channel) {
|
||||
logger.warn("[DeliverErrorGroupAlert] Channel not found or disabled", {
|
||||
channelId: payload.channelId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const errorLink = this.#buildErrorLink(
|
||||
channel.project.organization,
|
||||
channel.project,
|
||||
payload.error
|
||||
);
|
||||
|
||||
try {
|
||||
switch (channel.type) {
|
||||
case "EMAIL":
|
||||
await this.#sendEmail(channel, payload, errorLink);
|
||||
break;
|
||||
case "SLACK":
|
||||
await this.#sendSlack(channel, payload, errorLink);
|
||||
break;
|
||||
case "WEBHOOK":
|
||||
await this.#sendWebhook(channel, payload, errorLink);
|
||||
break;
|
||||
default:
|
||||
assertNever(channel.type);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof SkipRetryError) {
|
||||
logger.warn("[DeliverErrorGroupAlert] Skipping retry", {
|
||||
reason: (error as Error).message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
#buildErrorLink(
|
||||
organization: { slug: string },
|
||||
project: { slug: string },
|
||||
error: ErrorAlertPayload["error"]
|
||||
): string {
|
||||
return `${env.APP_ORIGIN}${v3ErrorPath(organization, project, { slug: error.environmentSlug }, { fingerprint: error.fingerprint })}`;
|
||||
}
|
||||
|
||||
#classificationLabel(classification: ErrorAlertClassification): string {
|
||||
switch (classification) {
|
||||
case "new_issue":
|
||||
return "New error";
|
||||
case "regression":
|
||||
return "Regression";
|
||||
case "unignored":
|
||||
return "Error resurfaced";
|
||||
}
|
||||
}
|
||||
|
||||
async #sendEmail(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: { name: string; organization: { title: string } };
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties);
|
||||
if (!emailProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse email properties", {
|
||||
issues: emailProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await sendAlertEmail({
|
||||
email: "alert-error-group",
|
||||
to: emailProperties.data.email,
|
||||
classification: payload.classification,
|
||||
taskIdentifier: payload.error.taskIdentifier,
|
||||
environment: payload.error.environmentName,
|
||||
error: {
|
||||
message: payload.error.errorMessage,
|
||||
type: payload.error.errorType,
|
||||
stackTrace: payload.error.sampleStackTrace || undefined,
|
||||
},
|
||||
occurrenceCount: payload.error.occurrenceCount,
|
||||
errorLink,
|
||||
organization: channel.project.organization.title,
|
||||
project: channel.project.name,
|
||||
});
|
||||
}
|
||||
|
||||
async #sendSlack(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: { organizationId: string; name: string; organization: { title: string } };
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties);
|
||||
if (!slackProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse slack properties", {
|
||||
issues: slackProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const integration = slackProperties.data.integrationId
|
||||
? await prisma.organizationIntegration.findFirst({
|
||||
where: {
|
||||
id: slackProperties.data.integrationId,
|
||||
organizationId: channel.project.organizationId,
|
||||
},
|
||||
include: { tokenReference: true },
|
||||
})
|
||||
: await prisma.organizationIntegration.findFirst({
|
||||
where: {
|
||||
service: "SLACK",
|
||||
organizationId: channel.project.organizationId,
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: { tokenReference: true },
|
||||
});
|
||||
|
||||
if (!integration || !isIntegrationForService(integration, "SLACK")) {
|
||||
logger.error("[DeliverErrorGroupAlert] Slack integration not found");
|
||||
return;
|
||||
}
|
||||
|
||||
const message = this.#buildErrorGroupSlackMessage(payload, errorLink, channel.project.name);
|
||||
|
||||
await this.#postSlackMessage(integration, {
|
||||
channel: slackProperties.data.channelId,
|
||||
...message,
|
||||
} as ChatPostMessageArguments);
|
||||
}
|
||||
|
||||
async #sendWebhook(
|
||||
channel: {
|
||||
type: ProjectAlertChannelType;
|
||||
properties: unknown;
|
||||
project: {
|
||||
id: string;
|
||||
externalRef: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
organizationId: string;
|
||||
organization: { slug: string; title: string };
|
||||
};
|
||||
},
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string
|
||||
): Promise<void> {
|
||||
const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties);
|
||||
if (!webhookProperties.success) {
|
||||
logger.error("[DeliverErrorGroupAlert] Failed to parse webhook properties", {
|
||||
issues: webhookProperties.error.issues,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const webhookPayload = generateErrorGroupWebhookPayload({
|
||||
classification: payload.classification,
|
||||
error: payload.error,
|
||||
organization: {
|
||||
id: channel.project.organizationId,
|
||||
slug: channel.project.organization.slug,
|
||||
name: channel.project.organization.title,
|
||||
},
|
||||
project: {
|
||||
id: channel.project.id,
|
||||
externalRef: channel.project.externalRef,
|
||||
slug: channel.project.slug,
|
||||
name: channel.project.name,
|
||||
},
|
||||
dashboardUrl: errorLink,
|
||||
});
|
||||
|
||||
const rawPayload = JSON.stringify(webhookPayload);
|
||||
const hashPayload = Buffer.from(rawPayload, "utf-8");
|
||||
const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret);
|
||||
const hmacSecret = Buffer.from(secret, "utf-8");
|
||||
const key = await subtle.importKey(
|
||||
"raw",
|
||||
hmacSecret,
|
||||
{ name: "HMAC", hash: "SHA-256" },
|
||||
false,
|
||||
["sign"]
|
||||
);
|
||||
const signature = await subtle.sign("HMAC", key, hashPayload);
|
||||
const signatureHex = Buffer.from(signature).toString("hex");
|
||||
|
||||
// Deliver via the SSRF-safe wrapper (see safeWebhookFetch.server.ts).
|
||||
const response = await safeWebhookFetch(webhookProperties.data.url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-trigger-signature-hmacsha256": signatureHex,
|
||||
},
|
||||
body: rawPayload,
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.info("[DeliverErrorGroupAlert] Failed to send webhook", {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
url: webhookProperties.data.url,
|
||||
});
|
||||
throw new Error(`Failed to send error group alert webhook to ${webhookProperties.data.url}`);
|
||||
}
|
||||
}
|
||||
|
||||
async #postSlackMessage(
|
||||
integration: OrganizationIntegrationForService<"SLACK">,
|
||||
message: ChatPostMessageArguments
|
||||
) {
|
||||
const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration(
|
||||
integration,
|
||||
{ forceBotToken: true }
|
||||
);
|
||||
|
||||
try {
|
||||
return await client.chat.postMessage({
|
||||
...message,
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isWebAPIRateLimitedError(error)) {
|
||||
throw new Error("Slack rate limited");
|
||||
}
|
||||
if (isWebAPIPlatformError(error)) {
|
||||
if (
|
||||
(error as WebAPIPlatformError).data.error === "invalid_blocks" ||
|
||||
(error as WebAPIPlatformError).data.error === "account_inactive"
|
||||
) {
|
||||
throw new SkipRetryError(`Slack: ${(error as WebAPIPlatformError).data.error}`);
|
||||
}
|
||||
throw new Error("Slack platform error");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
#buildErrorGroupSlackMessage(
|
||||
payload: ErrorAlertPayload,
|
||||
errorLink: string,
|
||||
projectName: string
|
||||
): { text: string; blocks: object[]; attachments: object[] } {
|
||||
const label = this.#classificationLabel(payload.classification);
|
||||
const errorType = payload.error.errorType || "Error";
|
||||
const task = payload.error.taskIdentifier;
|
||||
const envName = payload.error.environmentName;
|
||||
|
||||
return {
|
||||
text: `${label}: ${errorType} in ${task} [${envName}]`,
|
||||
blocks: [
|
||||
{
|
||||
type: "section",
|
||||
text: {
|
||||
type: "mrkdwn",
|
||||
text: `*${label} in ${task} [${envName}]*`,
|
||||
},
|
||||
},
|
||||
],
|
||||
attachments: [
|
||||
{
|
||||
color: "danger",
|
||||
blocks: [
|
||||
{
|
||||
type: "section",
|
||||
text: {
|
||||
type: "mrkdwn",
|
||||
text: this.#wrapInCodeBlock(
|
||||
payload.error.sampleStackTrace || payload.error.errorMessage
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "section",
|
||||
fields: [
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Task:*\n${task}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Environment:*\n${envName}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Project:*\n${projectName}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Occurrences:*\n${payload.error.occurrenceCount}`,
|
||||
},
|
||||
{
|
||||
type: "mrkdwn",
|
||||
text: `*Last seen:*\n${this.#formatTimestamp(new Date(Number(payload.error.lastSeen)))}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "actions",
|
||||
elements: [
|
||||
{
|
||||
type: "button",
|
||||
text: { type: "plain_text", text: "Investigate" },
|
||||
url: errorLink,
|
||||
style: "primary",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
#wrapInCodeBlock(text: string, maxLength = 3000) {
|
||||
const wrapperLength = 6; // ``` prefix + ``` suffix
|
||||
const truncationSuffix = "\n\n...truncated — check dashboard for full error";
|
||||
const innerMax = maxLength - wrapperLength;
|
||||
|
||||
const truncated =
|
||||
text.length > innerMax
|
||||
? text.slice(0, innerMax - truncationSuffix.length) + truncationSuffix
|
||||
: text;
|
||||
return `\`\`\`${truncated}\`\`\``;
|
||||
}
|
||||
|
||||
#formatTimestamp(date: Date): string {
|
||||
const unix = Math.floor(date.getTime() / 1000);
|
||||
const fallback =
|
||||
new Intl.DateTimeFormat("en-US", {
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
year: "numeric",
|
||||
hour: "numeric",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: true,
|
||||
timeZone: "UTC",
|
||||
}).format(date) + " UTC";
|
||||
return `<!date^${unix}^{date_short_pretty} {time_secs}|${fallback}>`;
|
||||
}
|
||||
}
|
||||
|
||||
function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError {
|
||||
return (error as WebAPIPlatformError).code === ErrorCode.PlatformError;
|
||||
}
|
||||
|
||||
function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError {
|
||||
return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError;
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
import { type ActiveErrorsSinceQueryResult } from "@internal/clickhouse";
|
||||
import {
|
||||
type ErrorGroupState,
|
||||
type PrismaClientOrTransaction,
|
||||
type ProjectAlertChannel,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import { $replica, prisma } from "~/db.server";
|
||||
import { ErrorAlertConfig } from "~/models/projectAlert.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
|
||||
type ErrorClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
interface AlertableError {
|
||||
classification: ErrorClassification;
|
||||
error: ActiveErrorsSinceQueryResult;
|
||||
environmentSlug: string;
|
||||
environmentName: string;
|
||||
}
|
||||
|
||||
interface ResolvedEnvironment {
|
||||
id: string;
|
||||
slug: string;
|
||||
type: RuntimeEnvironmentType;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
const DEFAULT_INTERVAL_MS = 300_000;
|
||||
|
||||
/**
|
||||
* For a project evalutes whether to send error alerts
|
||||
*
|
||||
* Alerts are sent if an error is
|
||||
* 1. A new issue
|
||||
* 2. A regression (was resolved and now back)
|
||||
* 3. Unignored (was ignored and is no longer)
|
||||
*
|
||||
* Unignored happens in 3 situations
|
||||
* 1. It was ignored with a future date, and that's now in the past
|
||||
* 2. It was ignored until reaching an error rate (e.g. 10/minute) and that has been exceeded
|
||||
* 3. It was ignored until reaching a total occurrence count (e.g. 1,000) and that has been exceeded
|
||||
*/
|
||||
export class ErrorAlertEvaluator {
|
||||
constructor(
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
protected readonly _replica: PrismaClientOrTransaction = $replica
|
||||
) {}
|
||||
|
||||
async evaluate(projectId: string, scheduledAt: number): Promise<void> {
|
||||
const nextScheduledAt = Date.now();
|
||||
|
||||
const channels = await this.resolveChannels(projectId);
|
||||
if (channels.length === 0) {
|
||||
logger.info("[ErrorAlertEvaluator] No active ERROR_GROUP channels, self-terminating", {
|
||||
projectId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const minIntervalMs = this.computeMinInterval(channels);
|
||||
const windowMs = nextScheduledAt - scheduledAt;
|
||||
|
||||
if (windowMs > minIntervalMs * 2) {
|
||||
logger.info("[ErrorAlertEvaluator] Large evaluation window (gap detected)", {
|
||||
projectId,
|
||||
scheduledAt,
|
||||
nextScheduledAt,
|
||||
windowMs,
|
||||
minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
const allEnvTypes = this.collectEnvironmentTypes(channels);
|
||||
|
||||
try {
|
||||
const [project, environments] = await Promise.all([
|
||||
this._replica.project.findFirst({
|
||||
where: { id: projectId },
|
||||
select: { organizationId: true },
|
||||
}),
|
||||
this.resolveEnvironments(projectId, allEnvTypes),
|
||||
]);
|
||||
|
||||
if (!project) {
|
||||
logger.error("[ErrorAlertEvaluator] Project not found", { projectId });
|
||||
return;
|
||||
}
|
||||
|
||||
if (environments.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const envIds = environments.map((e) => e.id);
|
||||
const envMap = new Map(environments.map((e) => [e.id, e]));
|
||||
const channelsByEnvId = this.buildChannelsByEnvId(channels, environments);
|
||||
|
||||
const activeErrors = await this.getActiveErrors(
|
||||
project.organizationId,
|
||||
projectId,
|
||||
envIds,
|
||||
scheduledAt
|
||||
);
|
||||
|
||||
if (activeErrors.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const states = await this.getErrorGroupStates(activeErrors);
|
||||
const stateMap = this.buildStateMap(states);
|
||||
|
||||
const occurrenceCounts = await this.getOccurrenceCountsSince(
|
||||
project.organizationId,
|
||||
projectId,
|
||||
envIds,
|
||||
scheduledAt
|
||||
);
|
||||
const occurrenceMap = this.buildOccurrenceMap(occurrenceCounts);
|
||||
|
||||
const alertableErrors: AlertableError[] = [];
|
||||
|
||||
for (const error of activeErrors) {
|
||||
const key = `${error.environment_id}:${error.task_identifier}:${error.error_fingerprint}`;
|
||||
const state = stateMap.get(key);
|
||||
const env = envMap.get(error.environment_id);
|
||||
const firstSeenMs = Number(error.first_seen);
|
||||
|
||||
const classification = this.classifyError(error, state, firstSeenMs, scheduledAt, {
|
||||
occurrencesSince: occurrenceMap.get(key) ?? 0,
|
||||
windowMs,
|
||||
totalOccurrenceCount: error.occurrence_count,
|
||||
});
|
||||
|
||||
if (classification) {
|
||||
alertableErrors.push({
|
||||
classification,
|
||||
error,
|
||||
environmentSlug: env?.slug ?? "",
|
||||
environmentName: env?.displayName ?? error.environment_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const alertable of alertableErrors) {
|
||||
const envChannels = channelsByEnvId.get(alertable.error.environment_id) ?? [];
|
||||
for (const channel of envChannels) {
|
||||
await alertsWorker.enqueue({
|
||||
id: `deliverErrorGroupAlert:${channel.id}:${alertable.error.error_fingerprint}:${scheduledAt}`,
|
||||
job: "v3.deliverErrorGroupAlert",
|
||||
payload: {
|
||||
channelId: channel.id,
|
||||
projectId,
|
||||
classification: alertable.classification,
|
||||
error: {
|
||||
fingerprint: alertable.error.error_fingerprint,
|
||||
environmentId: alertable.error.environment_id,
|
||||
environmentSlug: alertable.environmentSlug,
|
||||
environmentName: alertable.environmentName,
|
||||
taskIdentifier: alertable.error.task_identifier,
|
||||
errorType: alertable.error.error_type,
|
||||
errorMessage: alertable.error.error_message,
|
||||
sampleStackTrace: alertable.error.sample_stack_trace,
|
||||
firstSeen: alertable.error.first_seen,
|
||||
lastSeen: alertable.error.last_seen,
|
||||
occurrenceCount: alertable.error.occurrence_count,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await this.updateErrorGroupStates(
|
||||
alertableErrors,
|
||||
stateMap,
|
||||
project.organizationId,
|
||||
projectId
|
||||
);
|
||||
|
||||
logger.info("[ErrorAlertEvaluator] Evaluation complete", {
|
||||
projectId,
|
||||
activeErrors: activeErrors.length,
|
||||
alertableErrors: alertableErrors.length,
|
||||
deliveryJobsEnqueued: alertableErrors.reduce(
|
||||
(sum, a) => sum + (channelsByEnvId.get(a.error.environment_id)?.length ?? 0),
|
||||
0
|
||||
),
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("[ErrorAlertEvaluator] Evaluation failed, will retry on next cycle", {
|
||||
projectId,
|
||||
error,
|
||||
});
|
||||
} finally {
|
||||
await this.selfChain(projectId, nextScheduledAt, minIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private classifyError(
|
||||
error: ActiveErrorsSinceQueryResult,
|
||||
state: ErrorGroupState | undefined,
|
||||
firstSeenMs: number,
|
||||
scheduledAt: number,
|
||||
thresholdContext: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
|
||||
): ErrorClassification | null {
|
||||
if (!state) {
|
||||
return firstSeenMs > scheduledAt ? "new_issue" : null;
|
||||
}
|
||||
|
||||
switch (state.status) {
|
||||
case "UNRESOLVED":
|
||||
return null;
|
||||
|
||||
case "RESOLVED": {
|
||||
if (!state.resolvedAt) return null;
|
||||
const lastSeenMs = Number(error.last_seen);
|
||||
return lastSeenMs > state.resolvedAt.getTime() ? "regression" : null;
|
||||
}
|
||||
|
||||
case "IGNORED":
|
||||
return this.isIgnoreBreached(state, thresholdContext) ? "unignored" : null;
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isIgnoreBreached(
|
||||
state: ErrorGroupState,
|
||||
context: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
|
||||
): boolean {
|
||||
if (state.ignoredUntil && state.ignoredUntil.getTime() <= Date.now()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
state.ignoredUntilOccurrenceRate !== null &&
|
||||
state.ignoredUntilOccurrenceRate !== undefined
|
||||
) {
|
||||
const windowMinutes = Math.max(context.windowMs / 60_000, 1);
|
||||
const rate = context.occurrencesSince / windowMinutes;
|
||||
if (rate > state.ignoredUntilOccurrenceRate) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (state.ignoredUntilTotalOccurrences != null && state.ignoredAtOccurrenceCount != null) {
|
||||
const occurrencesSinceIgnored =
|
||||
context.totalOccurrenceCount - Number(state.ignoredAtOccurrenceCount);
|
||||
if (occurrencesSinceIgnored >= state.ignoredUntilTotalOccurrences) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async resolveChannels(projectId: string): Promise<ProjectAlertChannel[]> {
|
||||
return this._replica.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
alertTypes: { has: "ERROR_GROUP" },
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private computeMinInterval(channels: ProjectAlertChannel[]): number {
|
||||
let min = DEFAULT_INTERVAL_MS;
|
||||
for (const ch of channels) {
|
||||
const config = ErrorAlertConfig.safeParse(ch.errorAlertConfig);
|
||||
if (config.success) {
|
||||
min = Math.min(min, config.data.evaluationIntervalMs);
|
||||
}
|
||||
}
|
||||
return min;
|
||||
}
|
||||
|
||||
private collectEnvironmentTypes(channels: ProjectAlertChannel[]): RuntimeEnvironmentType[] {
|
||||
const types = new Set<RuntimeEnvironmentType>();
|
||||
for (const ch of channels) {
|
||||
for (const t of ch.environmentTypes) {
|
||||
types.add(t);
|
||||
}
|
||||
}
|
||||
return Array.from(types);
|
||||
}
|
||||
|
||||
private async resolveEnvironments(
|
||||
projectId: string,
|
||||
types: RuntimeEnvironmentType[]
|
||||
): Promise<ResolvedEnvironment[]> {
|
||||
const envs = await this._replica.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
type: { in: types },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
slug: true,
|
||||
branchName: true,
|
||||
},
|
||||
});
|
||||
|
||||
return envs.map((e) => ({
|
||||
id: e.id,
|
||||
slug: e.slug,
|
||||
type: e.type,
|
||||
displayName: e.branchName ?? e.slug,
|
||||
}));
|
||||
}
|
||||
|
||||
private buildChannelsByEnvId(
|
||||
channels: ProjectAlertChannel[],
|
||||
environments: ResolvedEnvironment[]
|
||||
): Map<string, ProjectAlertChannel[]> {
|
||||
const result = new Map<string, ProjectAlertChannel[]>();
|
||||
for (const env of environments) {
|
||||
const matching = channels.filter((ch) => ch.environmentTypes.includes(env.type));
|
||||
if (matching.length > 0) {
|
||||
result.set(env.id, matching);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async getActiveErrors(
|
||||
organizationId: string,
|
||||
projectId: string,
|
||||
envIds: string[],
|
||||
scheduledAt: number
|
||||
): Promise<ActiveErrorsSinceQueryResult[]> {
|
||||
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"query"
|
||||
);
|
||||
const qb = queryClickhouse.errors.activeErrorsSinceQueryBuilder();
|
||||
qb.where("organization_id = {organizationId: String}", { organizationId });
|
||||
qb.where("project_id = {projectId: String}", { projectId });
|
||||
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
|
||||
qb.groupBy("environment_id, task_identifier, error_fingerprint");
|
||||
qb.having("toInt64(last_seen) > {scheduledAt: Int64}", {
|
||||
scheduledAt,
|
||||
});
|
||||
|
||||
const [err, results] = await qb.execute();
|
||||
if (err) {
|
||||
logger.error("[ErrorAlertEvaluator] Failed to query active errors", { error: err });
|
||||
return [];
|
||||
}
|
||||
return results ?? [];
|
||||
}
|
||||
|
||||
private async getErrorGroupStates(
|
||||
activeErrors: ActiveErrorsSinceQueryResult[]
|
||||
): Promise<ErrorGroupState[]> {
|
||||
if (activeErrors.length === 0) return [];
|
||||
|
||||
return this._replica.errorGroupState.findMany({
|
||||
where: {
|
||||
OR: activeErrors.map((e) => ({
|
||||
environmentId: e.environment_id,
|
||||
taskIdentifier: e.task_identifier,
|
||||
errorFingerprint: e.error_fingerprint,
|
||||
})),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private buildStateMap(states: ErrorGroupState[]): Map<string, ErrorGroupState> {
|
||||
const map = new Map<string, ErrorGroupState>();
|
||||
for (const s of states) {
|
||||
map.set(`${s.environmentId}:${s.taskIdentifier}:${s.errorFingerprint}`, s);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private async getOccurrenceCountsSince(
|
||||
organizationId: string,
|
||||
projectId: string,
|
||||
envIds: string[],
|
||||
scheduledAt: number
|
||||
): Promise<
|
||||
Array<{
|
||||
environment_id: string;
|
||||
task_identifier: string;
|
||||
error_fingerprint: string;
|
||||
occurrences_since: number;
|
||||
}>
|
||||
> {
|
||||
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"query"
|
||||
);
|
||||
const qb = queryClickhouse.errors.occurrenceCountsSinceQueryBuilder();
|
||||
qb.where("organization_id = {organizationId: String}", { organizationId });
|
||||
qb.where("project_id = {projectId: String}", { projectId });
|
||||
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
|
||||
qb.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({scheduledAt: Int64}))", {
|
||||
scheduledAt,
|
||||
});
|
||||
qb.groupBy("environment_id, task_identifier, error_fingerprint");
|
||||
|
||||
const [err, results] = await qb.execute();
|
||||
if (err) {
|
||||
logger.error("[ErrorAlertEvaluator] Failed to query occurrence counts", { error: err });
|
||||
return [];
|
||||
}
|
||||
return results ?? [];
|
||||
}
|
||||
|
||||
private buildOccurrenceMap(
|
||||
counts: Array<{
|
||||
environment_id: string;
|
||||
task_identifier: string;
|
||||
error_fingerprint: string;
|
||||
occurrences_since: number;
|
||||
}>
|
||||
): Map<string, number> {
|
||||
const map = new Map<string, number>();
|
||||
for (const c of counts) {
|
||||
map.set(
|
||||
`${c.environment_id}:${c.task_identifier}:${c.error_fingerprint}`,
|
||||
c.occurrences_since
|
||||
);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private async updateErrorGroupStates(
|
||||
alertableErrors: AlertableError[],
|
||||
stateMap: Map<string, ErrorGroupState>,
|
||||
organizationId: string,
|
||||
projectId: string
|
||||
): Promise<void> {
|
||||
for (const alertable of alertableErrors) {
|
||||
const key = `${alertable.error.environment_id}:${alertable.error.task_identifier}:${alertable.error.error_fingerprint}`;
|
||||
const state = stateMap.get(key);
|
||||
|
||||
if (state) {
|
||||
await this._prisma.errorGroupState.update({
|
||||
where: { id: state.id },
|
||||
data: {
|
||||
status: "UNRESOLVED",
|
||||
ignoredUntil: null,
|
||||
ignoredUntilOccurrenceRate: null,
|
||||
ignoredUntilTotalOccurrences: null,
|
||||
ignoredAtOccurrenceCount: null,
|
||||
ignoredAt: null,
|
||||
ignoredReason: null,
|
||||
ignoredByUserId: null,
|
||||
resolvedAt: null,
|
||||
resolvedInVersion: null,
|
||||
resolvedBy: null,
|
||||
},
|
||||
});
|
||||
} else if (alertable.classification === "new_issue") {
|
||||
await this._prisma.errorGroupState.create({
|
||||
data: {
|
||||
organizationId,
|
||||
projectId,
|
||||
environmentId: alertable.error.environment_id,
|
||||
taskIdentifier: alertable.error.task_identifier,
|
||||
errorFingerprint: alertable.error.error_fingerprint,
|
||||
status: "UNRESOLVED",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async selfChain(
|
||||
projectId: string,
|
||||
nextScheduledAt: number,
|
||||
intervalMs: number
|
||||
): Promise<void> {
|
||||
await alertsWorker.enqueue({
|
||||
id: `evaluateErrorAlerts:${projectId}`,
|
||||
job: "v3.evaluateErrorAlerts",
|
||||
payload: {
|
||||
projectId,
|
||||
scheduledAt: nextScheduledAt,
|
||||
},
|
||||
availableAt: new Date(nextScheduledAt + intervalMs),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import type { ErrorWebhook } from "@trigger.dev/core/v3/schemas";
|
||||
|
||||
export type ErrorAlertClassification = "new_issue" | "regression" | "unignored";
|
||||
|
||||
export type ErrorGroupAlertData = {
|
||||
classification: ErrorAlertClassification;
|
||||
error: {
|
||||
fingerprint: string;
|
||||
environmentId: string;
|
||||
environmentName: string;
|
||||
taskIdentifier: string;
|
||||
errorType: string;
|
||||
errorMessage: string;
|
||||
sampleStackTrace: string;
|
||||
firstSeen: string;
|
||||
lastSeen: string;
|
||||
occurrenceCount: number;
|
||||
};
|
||||
organization: {
|
||||
id: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
};
|
||||
project: {
|
||||
id: string;
|
||||
externalRef: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
};
|
||||
dashboardUrl: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generates a webhook payload for an error group alert that conforms to the
|
||||
* ErrorWebhook schema from @trigger.dev/core/v3/schemas
|
||||
*/
|
||||
export function generateErrorGroupWebhookPayload(data: ErrorGroupAlertData): ErrorWebhook {
|
||||
return {
|
||||
id: nanoid(),
|
||||
created: new Date(),
|
||||
webhookVersion: "2025-01-01",
|
||||
type: "alert.error" as const,
|
||||
object: {
|
||||
classification: data.classification,
|
||||
error: {
|
||||
fingerprint: data.error.fingerprint,
|
||||
type: data.error.errorType,
|
||||
message: data.error.errorMessage,
|
||||
stackTrace: data.error.sampleStackTrace || undefined,
|
||||
firstSeen: new Date(Number(data.error.firstSeen)),
|
||||
lastSeen: new Date(Number(data.error.lastSeen)),
|
||||
occurrenceCount: data.error.occurrenceCount,
|
||||
taskIdentifier: data.error.taskIdentifier,
|
||||
},
|
||||
environment: {
|
||||
id: data.error.environmentId,
|
||||
name: data.error.environmentName,
|
||||
},
|
||||
organization: {
|
||||
id: data.organization.id,
|
||||
slug: data.organization.slug,
|
||||
name: data.organization.name,
|
||||
},
|
||||
project: {
|
||||
id: data.project.id,
|
||||
ref: data.project.externalRef,
|
||||
slug: data.project.slug,
|
||||
name: data.project.name,
|
||||
},
|
||||
dashboardUrl: data.dashboardUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import type {
|
||||
ProjectAlertChannel,
|
||||
ProjectAlertType,
|
||||
WorkerDeployment,
|
||||
} from "@trigger.dev/database";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { DeliverAlertService } from "./deliverAlert.server";
|
||||
|
||||
export class PerformDeploymentAlertsService extends BaseService {
|
||||
public async call(deploymentId: string) {
|
||||
const deployment = await this._prisma.workerDeployment.findFirst({
|
||||
where: { id: deploymentId },
|
||||
include: {
|
||||
environment: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!deployment) {
|
||||
return;
|
||||
}
|
||||
|
||||
const alertType =
|
||||
deployment.status === "DEPLOYED" ? "DEPLOYMENT_SUCCESS" : "DEPLOYMENT_FAILURE";
|
||||
|
||||
// Find all the alert channels
|
||||
const alertChannels = await this._prisma.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId: deployment.projectId,
|
||||
alertTypes: {
|
||||
has: alertType,
|
||||
},
|
||||
environmentTypes: {
|
||||
has: deployment.environment.type,
|
||||
},
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const alertChannel of alertChannels) {
|
||||
await this.#createAndSendAlert(alertChannel, deployment, alertType);
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndSendAlert(
|
||||
alertChannel: ProjectAlertChannel,
|
||||
deployment: WorkerDeployment,
|
||||
alertType: ProjectAlertType
|
||||
) {
|
||||
await DeliverAlertService.createAndSendAlert(
|
||||
{
|
||||
channelId: alertChannel.id,
|
||||
channelType: alertChannel.type,
|
||||
projectId: deployment.projectId,
|
||||
environmentId: deployment.environmentId,
|
||||
alertType,
|
||||
deploymentId: deployment.id,
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
}
|
||||
|
||||
static async enqueue(deploymentId: string, runAt?: Date) {
|
||||
return await alertsWorker.enqueue({
|
||||
id: `performDeploymentAlerts:${deploymentId}`,
|
||||
job: "v3.performDeploymentAlerts",
|
||||
payload: { deploymentId },
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { type RunStore } from "@internal/run-store";
|
||||
import { type Prisma, type ProjectAlertChannel } from "@trigger.dev/database";
|
||||
import { type PrismaClientOrTransaction, type prisma } from "~/db.server";
|
||||
import { alertsWorker } from "~/v3/alertsWorker.server";
|
||||
import type { ControlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { controlPlaneResolver as defaultControlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { DeliverAlertService } from "./deliverAlert.server";
|
||||
|
||||
// The alert hydration reads only run-ops scalars (id/projectId/runtimeEnvironmentId); the env's
|
||||
// type (and its parent's) is resolved via the control-plane resolver so the run-ops DB can split
|
||||
// without a cross-provider join. The prior `lockedBy` + `runtimeEnvironment` includes were unused.
|
||||
type FoundRun = Prisma.Result<
|
||||
typeof prisma.taskRun,
|
||||
{ select: { id: true; projectId: true; runtimeEnvironmentId: true } },
|
||||
"findUniqueOrThrow"
|
||||
>;
|
||||
|
||||
export class PerformTaskRunAlertsService extends BaseService {
|
||||
#controlPlaneResolver: ControlPlaneResolver;
|
||||
|
||||
constructor(
|
||||
opts: {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
replica?: PrismaClientOrTransaction;
|
||||
runStore?: RunStore;
|
||||
controlPlaneResolver?: ControlPlaneResolver;
|
||||
} = {}
|
||||
) {
|
||||
super(opts.prisma, opts.replica, opts.runStore);
|
||||
this.#controlPlaneResolver = opts.controlPlaneResolver ?? defaultControlPlaneResolver;
|
||||
}
|
||||
|
||||
public async call(runId: string) {
|
||||
const run = await this.runStore.findRun(
|
||||
{ id: runId },
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
projectId: true,
|
||||
runtimeEnvironmentId: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
return;
|
||||
}
|
||||
|
||||
const env = await this.#controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId);
|
||||
|
||||
if (!env) {
|
||||
return;
|
||||
}
|
||||
|
||||
const alertChannels = await this._prisma.projectAlertChannel.findMany({
|
||||
where: {
|
||||
projectId: run.projectId,
|
||||
alertTypes: {
|
||||
has: "TASK_RUN",
|
||||
},
|
||||
environmentTypes: {
|
||||
has: env.parentEnvironmentType ?? env.type,
|
||||
},
|
||||
enabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const alertChannel of alertChannels) {
|
||||
await this.#createAndSendAlert(alertChannel, run);
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndSendAlert(alertChannel: ProjectAlertChannel, run: FoundRun) {
|
||||
await DeliverAlertService.createAndSendAlert(
|
||||
{
|
||||
channelId: alertChannel.id,
|
||||
channelType: alertChannel.type,
|
||||
projectId: run.projectId,
|
||||
environmentId: run.runtimeEnvironmentId,
|
||||
alertType: "TASK_RUN",
|
||||
taskRunId: run.id,
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
}
|
||||
|
||||
static async enqueue(runId: string, runAt?: Date) {
|
||||
return await alertsWorker.enqueue({
|
||||
id: `performTaskRunAlerts:${runId}`,
|
||||
job: "v3.performTaskRunAlerts",
|
||||
payload: { runId },
|
||||
availableAt: runAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { promises as dnsPromises } from "node:dns";
|
||||
import type { LookupFunction } from "node:net";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import {
|
||||
assertAddressAllowed,
|
||||
assertSafeWebhookUrl,
|
||||
assertSafeWebhookUrlLexical,
|
||||
UnsafeWebhookUrlError,
|
||||
} from "./safeWebhookUrl.server";
|
||||
|
||||
/**
|
||||
* `fetch`-like wrapper for delivering user-supplied webhook URLs. The lexical
|
||||
* check is shared with the storage-time gate (`assertSafeWebhookUrlLexical`).
|
||||
*
|
||||
* Validation is bound to the actual connection: the request goes through
|
||||
* `node:http`/`node:https` with a custom DNS `lookup` that validates every
|
||||
* resolved address before the socket connects, so the connected address is the
|
||||
* one that was checked. Redirects are followed manually and re-validated per
|
||||
* hop, capped at `MAX_REDIRECTS`.
|
||||
*/
|
||||
|
||||
// Re-exported so callers/tests don't reach into the underlying module.
|
||||
export { assertSafeWebhookUrl, assertSafeWebhookUrlLexical, UnsafeWebhookUrlError };
|
||||
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
export type SafeWebhookFetchInit = {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: string | Buffer;
|
||||
signal?: AbortSignal;
|
||||
redirectLimit?: number;
|
||||
};
|
||||
|
||||
// DNS lookup that validates every resolved address before handing it to the
|
||||
// connector; any unsafe address fails the whole lookup. On error we pass an
|
||||
// empty address list, which net ignores when err is set.
|
||||
const safeLookup: LookupFunction = (hostname, options, callback) => {
|
||||
dnsPromises
|
||||
.lookup(hostname, {
|
||||
all: true,
|
||||
family: options.family,
|
||||
hints: options.hints,
|
||||
verbatim: options.verbatim,
|
||||
})
|
||||
.then((addresses) => {
|
||||
try {
|
||||
for (const { address, family } of addresses) {
|
||||
assertAddressAllowed(address, family);
|
||||
}
|
||||
} catch (err) {
|
||||
callback(err as NodeJS.ErrnoException, []);
|
||||
return;
|
||||
}
|
||||
if (options.all) {
|
||||
callback(null, addresses);
|
||||
} else {
|
||||
callback(null, addresses[0].address, addresses[0].family);
|
||||
}
|
||||
})
|
||||
.catch((err) => callback(err as NodeJS.ErrnoException, []));
|
||||
};
|
||||
|
||||
// Single request with no redirect following, using the validating lookup. The
|
||||
// response body is drained and discarded (callers only need status / headers),
|
||||
// which also frees the socket.
|
||||
function requestOnce(urlStr: string, init: SafeWebhookFetchInit): Promise<Response> {
|
||||
const url = new URL(urlStr);
|
||||
const mod = url.protocol === "https:" ? https : http;
|
||||
// Set Content-Length explicitly (as fetch does for string/Buffer bodies)
|
||||
// rather than falling back to chunked transfer-encoding, which some
|
||||
// webhook receivers reject.
|
||||
const headers: Record<string, string> = { ...(init.headers ?? {}) };
|
||||
if (
|
||||
init.body != null &&
|
||||
headers["content-length"] === undefined &&
|
||||
headers["Content-Length"] === undefined
|
||||
) {
|
||||
headers["content-length"] = String(Buffer.byteLength(init.body));
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = mod.request(
|
||||
url,
|
||||
{
|
||||
method: init.method ?? "GET",
|
||||
headers,
|
||||
lookup: safeLookup,
|
||||
signal: init.signal,
|
||||
},
|
||||
(res) => {
|
||||
res.on("data", () => {});
|
||||
res.on("end", () => {
|
||||
const responseHeaders = new Headers();
|
||||
for (const [key, value] of Object.entries(res.headers)) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const v of value) responseHeaders.append(key, v);
|
||||
} else if (value !== undefined) {
|
||||
responseHeaders.set(key, value);
|
||||
}
|
||||
}
|
||||
resolve(
|
||||
new Response(null, {
|
||||
status: res.statusCode ?? 502,
|
||||
statusText: res.statusMessage ?? "",
|
||||
headers: responseHeaders,
|
||||
})
|
||||
);
|
||||
});
|
||||
res.on("error", reject);
|
||||
}
|
||||
);
|
||||
req.on("error", reject);
|
||||
if (init.body != null) {
|
||||
req.write(init.body);
|
||||
}
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Tenant-supplied-URL fetch with connection-bound SSRF validation and manual,
|
||||
* per-hop redirect validation.
|
||||
*/
|
||||
export async function safeWebhookFetch(
|
||||
rawUrl: string,
|
||||
init: SafeWebhookFetchInit = {}
|
||||
): Promise<Response> {
|
||||
let nextUrl = assertSafeWebhookUrlLexical(rawUrl).href;
|
||||
const limit = init.redirectLimit ?? MAX_REDIRECTS;
|
||||
|
||||
for (let hop = 0; hop <= limit; hop++) {
|
||||
const response = await requestOnce(nextUrl, init);
|
||||
if (response.status < 300 || response.status >= 400) {
|
||||
return response;
|
||||
}
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return response;
|
||||
if (hop === limit) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Refusing to deliver webhook to ${nextUrl}: exceeded redirect limit (${limit}) following ${location}`
|
||||
);
|
||||
}
|
||||
const target = new URL(location, nextUrl);
|
||||
try {
|
||||
nextUrl = assertSafeWebhookUrlLexical(target.href).href;
|
||||
} catch (err) {
|
||||
logger.warn("Refusing to follow webhook redirect", {
|
||||
from: nextUrl,
|
||||
to: target.href,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// Unreachable — the loop always returns or throws.
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Refusing to deliver webhook to ${nextUrl}: exhausted redirect loop`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Validator for user-supplied webhook URLs that the server fetches later
|
||||
// (alert channels, error-group webhooks). Rejects non-http(s) schemes and
|
||||
// private/loopback/link-local/reserved hosts.
|
||||
//
|
||||
// Two entry points:
|
||||
// - `assertSafeWebhookUrlLexical` — sync, no network. Runs on every
|
||||
// delivery hop; the connect-time bound lookup below is authoritative.
|
||||
// - `assertSafeWebhookUrl` — storage-time gate: lexical check plus a
|
||||
// best-effort DNS resolution for early, friendly rejection.
|
||||
//
|
||||
// The authoritative guard is at delivery time: `safeWebhookFetch` binds
|
||||
// validation into the connection's own DNS lookup, so the address actually
|
||||
// connected to is the one that was checked.
|
||||
|
||||
import { promises as dnsPromises } from "node:dns";
|
||||
|
||||
export class UnsafeWebhookUrlError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "UnsafeWebhookUrlError";
|
||||
}
|
||||
}
|
||||
|
||||
function isUnsafeIPv4(host: string): boolean {
|
||||
// Reject if the host parses as a 4-octet IPv4 in any of the unsafe ranges.
|
||||
const parts = host.split(".");
|
||||
if (parts.length !== 4) return false;
|
||||
const nums = parts.map((p) => Number(p));
|
||||
if (nums.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return false;
|
||||
const [a, b] = nums;
|
||||
// 0.0.0.0/8 (unspecified)
|
||||
if (a === 0) return true;
|
||||
// 127/8 loopback
|
||||
if (a === 127) return true;
|
||||
// 10/8
|
||||
if (a === 10) return true;
|
||||
// 172.16/12
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
// 192.168/16
|
||||
if (a === 192 && b === 168) return true;
|
||||
// 169.254/16 link-local
|
||||
if (a === 169 && b === 254) return true;
|
||||
// 100.64/10 carrier-grade NAT
|
||||
if (a === 100 && b >= 64 && b <= 127) return true;
|
||||
// 224/4 multicast
|
||||
if (a >= 224 && a <= 239) return true;
|
||||
// 240/4 reserved
|
||||
if (a >= 240) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isUnsafeIPv6(host: string): boolean {
|
||||
// URL.hostname keeps the brackets for IPv6 literals ([::1]); DNS results
|
||||
// and IP literals elsewhere are unbracketed. Strip brackets so both work.
|
||||
const lower = (
|
||||
host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host
|
||||
).toLowerCase();
|
||||
// loopback
|
||||
if (lower === "::1") return true;
|
||||
// unspecified
|
||||
if (lower === "::" || lower === "::0" || lower === "0:0:0:0:0:0:0:0") return true;
|
||||
// link-local fe80::/10
|
||||
if (/^fe[89ab][0-9a-f]?:/.test(lower)) return true;
|
||||
// ULA fc00::/7
|
||||
if (/^f[cd][0-9a-f]{2}:/.test(lower)) return true;
|
||||
// multicast ff00::/8
|
||||
if (lower.startsWith("ff")) return true;
|
||||
// IPv4-mapped, dotted form: ::ffff:a.b.c.d
|
||||
const mappedDotted = lower.match(/^::ffff:([0-9.]+)$/);
|
||||
if (mappedDotted && isUnsafeIPv4(mappedDotted[1])) return true;
|
||||
// IPv4-mapped, hex form: ::ffff:7f00:1 (how Node normalizes ::ffff:127.0.0.1).
|
||||
// The two trailing hextets encode the 32-bit IPv4 address.
|
||||
const mappedHex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
||||
if (mappedHex) {
|
||||
const hi = parseInt(mappedHex[1], 16);
|
||||
const lo = parseInt(mappedHex[2], 16);
|
||||
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
||||
if (isUnsafeIPv4(ipv4)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if a resolved IP address falls in a disallowed range. Exposed so the
|
||||
* delivery-time connector can validate the actual address it connects to.
|
||||
*/
|
||||
export function assertAddressAllowed(address: string, family: number): void {
|
||||
if (family === 4 && isUnsafeIPv4(address)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL resolves to a private/loopback/link-local address: ${address}`
|
||||
);
|
||||
}
|
||||
if (family === 6 && isUnsafeIPv6(address)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL resolves to a private/loopback/link-local IPv6 address: ${address}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isUnsafeHostname(host: string): boolean {
|
||||
const lower = host.toLowerCase();
|
||||
if (lower === "localhost" || lower.endsWith(".localhost")) return true;
|
||||
if (lower === "internal" || lower.endsWith(".internal")) return true;
|
||||
if (lower === "local" || lower.endsWith(".local")) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isIPLiteral(host: string): boolean {
|
||||
// Strip brackets: URL.hostname keeps them for IPv6 literals ([::1]).
|
||||
const bare = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
||||
// IPv4: four dot-separated 0-255 octets.
|
||||
if (/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bare)) return true;
|
||||
// IPv6: at least one `:` and only hex / `:` / `.` (the `.` allows
|
||||
// IPv4-mapped notation like ::ffff:1.2.3.4).
|
||||
if (bare.includes(":") && /^[0-9a-fA-F:.]+$/.test(bare)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort storage-time DNS check: resolve `hostname` and throw if a
|
||||
* returned address is unsafe. Not a security boundary — resolution failures
|
||||
* don't block the save, since delivery re-validates at connect time.
|
||||
*/
|
||||
async function assertResolvedAddressesSafe(hostname: string): Promise<void> {
|
||||
let addresses: Array<{ address: string; family: number }>;
|
||||
try {
|
||||
addresses = await dnsPromises.lookup(hostname, { all: true });
|
||||
} catch {
|
||||
// Unresolvable right now — don't block the save; connect-time is authoritative.
|
||||
return;
|
||||
}
|
||||
for (const { address, family } of addresses) {
|
||||
assertAddressAllowed(address, family);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous, no-network SSRF check: scheme allow-list plus IP-literal
|
||||
* and hostname range checks. Used on every delivery hop, where the
|
||||
* connect-time bound lookup is the authoritative range check.
|
||||
*/
|
||||
export function assertSafeWebhookUrlLexical(rawUrl: string): URL {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new UnsafeWebhookUrlError("Webhook URL is not a valid URL");
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new UnsafeWebhookUrlError(`Webhook URL must use http or https (got ${parsed.protocol})`);
|
||||
}
|
||||
const host = parsed.hostname;
|
||||
if (!host) {
|
||||
throw new UnsafeWebhookUrlError("Webhook URL must have a hostname");
|
||||
}
|
||||
if (isUnsafeHostname(host)) {
|
||||
throw new UnsafeWebhookUrlError(`Webhook URL host is not allowed: ${host}`);
|
||||
}
|
||||
if (isUnsafeIPv4(host)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL points at a private/loopback/link-local address: ${host}`
|
||||
);
|
||||
}
|
||||
if (isUnsafeIPv6(host)) {
|
||||
throw new UnsafeWebhookUrlError(
|
||||
`Webhook URL points at a private/loopback/link-local IPv6 address: ${host}`
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage-time gate: lexical check plus a best-effort DNS resolution of
|
||||
* registrable domains, for early rejection before storage.
|
||||
*/
|
||||
export async function assertSafeWebhookUrl(rawUrl: string): Promise<URL> {
|
||||
const parsed = assertSafeWebhookUrlLexical(rawUrl);
|
||||
if (!isIPLiteral(parsed.hostname)) {
|
||||
await assertResolvedAddressesSafe(parsed.hostname);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
Reference in New Issue
Block a user