triggerdotdev--trigger.dev/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts

import { type ActiveErrorsSinceQueryResult } from "@internal/clickhouse";
import {
  type ErrorGroupState,
  type PrismaClientOrTransaction,
  type ProjectAlertChannel,
  type RuntimeEnvironmentType,
} from "@trigger.dev/database";
import { $replica, prisma } from "~/db.server";
import { ErrorAlertConfig } from "~/models/projectAlert.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { logger } from "~/services/logger.server";
import { alertsWorker } from "~/v3/alertsWorker.server";

type ErrorClassification = "new_issue" | "regression" | "unignored";

interface AlertableError {
  classification: ErrorClassification;
  error: ActiveErrorsSinceQueryResult;
  environmentSlug: string;
  environmentName: string;
}

interface ResolvedEnvironment {
  id: string;
  slug: string;
  type: RuntimeEnvironmentType;
  displayName: string;
}

const DEFAULT_INTERVAL_MS = 300_000;

/**
 * For a project evalutes whether to send error alerts
 *
 * Alerts are sent if an error is
 * 1. A new issue
 * 2. A regression (was resolved and now back)
 * 3. Unignored (was ignored and is no longer)
 *
 * Unignored happens in 3 situations
 * 1. It was ignored with a future date, and that's now in the past
 * 2. It was ignored until reaching an error rate (e.g. 10/minute) and that has been exceeded
 * 3. It was ignored until reaching a total occurrence count (e.g. 1,000) and that has been exceeded
 */
export class ErrorAlertEvaluator {
  constructor(
    protected readonly _prisma: PrismaClientOrTransaction = prisma,
    protected readonly _replica: PrismaClientOrTransaction = $replica
  ) {}

  async evaluate(projectId: string, scheduledAt: number): Promise<void> {
    const nextScheduledAt = Date.now();

    const channels = await this.resolveChannels(projectId);
    if (channels.length === 0) {
      logger.info("[ErrorAlertEvaluator] No active ERROR_GROUP channels, self-terminating", {
        projectId,
      });
      return;
    }

    const minIntervalMs = this.computeMinInterval(channels);
    const windowMs = nextScheduledAt - scheduledAt;

    if (windowMs > minIntervalMs * 2) {
      logger.info("[ErrorAlertEvaluator] Large evaluation window (gap detected)", {
        projectId,
        scheduledAt,
        nextScheduledAt,
        windowMs,
        minIntervalMs,
      });
    }

    const allEnvTypes = this.collectEnvironmentTypes(channels);

    try {
      const [project, environments] = await Promise.all([
        this._replica.project.findFirst({
          where: { id: projectId },
          select: { organizationId: true },
        }),
        this.resolveEnvironments(projectId, allEnvTypes),
      ]);

      if (!project) {
        logger.error("[ErrorAlertEvaluator] Project not found", { projectId });
        return;
      }

      if (environments.length === 0) {
        return;
      }

      const envIds = environments.map((e) => e.id);
      const envMap = new Map(environments.map((e) => [e.id, e]));
      const channelsByEnvId = this.buildChannelsByEnvId(channels, environments);

      const activeErrors = await this.getActiveErrors(
        project.organizationId,
        projectId,
        envIds,
        scheduledAt
      );

      if (activeErrors.length === 0) {
        return;
      }

      const states = await this.getErrorGroupStates(activeErrors);
      const stateMap = this.buildStateMap(states);

      const occurrenceCounts = await this.getOccurrenceCountsSince(
        project.organizationId,
        projectId,
        envIds,
        scheduledAt
      );
      const occurrenceMap = this.buildOccurrenceMap(occurrenceCounts);

      const alertableErrors: AlertableError[] = [];

      for (const error of activeErrors) {
        const key = `${error.environment_id}:${error.task_identifier}:${error.error_fingerprint}`;
        const state = stateMap.get(key);
        const env = envMap.get(error.environment_id);
        const firstSeenMs = Number(error.first_seen);

        const classification = this.classifyError(error, state, firstSeenMs, scheduledAt, {
          occurrencesSince: occurrenceMap.get(key) ?? 0,
          windowMs,
          totalOccurrenceCount: error.occurrence_count,
        });

        if (classification) {
          alertableErrors.push({
            classification,
            error,
            environmentSlug: env?.slug ?? "",
            environmentName: env?.displayName ?? error.environment_id,
          });
        }
      }

      for (const alertable of alertableErrors) {
        const envChannels = channelsByEnvId.get(alertable.error.environment_id) ?? [];
        for (const channel of envChannels) {
          await alertsWorker.enqueue({
            id: `deliverErrorGroupAlert:${channel.id}:${alertable.error.error_fingerprint}:${scheduledAt}`,
            job: "v3.deliverErrorGroupAlert",
            payload: {
              channelId: channel.id,
              projectId,
              classification: alertable.classification,
              error: {
                fingerprint: alertable.error.error_fingerprint,
                environmentId: alertable.error.environment_id,
                environmentSlug: alertable.environmentSlug,
                environmentName: alertable.environmentName,
                taskIdentifier: alertable.error.task_identifier,
                errorType: alertable.error.error_type,
                errorMessage: alertable.error.error_message,
                sampleStackTrace: alertable.error.sample_stack_trace,
                firstSeen: alertable.error.first_seen,
                lastSeen: alertable.error.last_seen,
                occurrenceCount: alertable.error.occurrence_count,
              },
            },
          });
        }
      }

      await this.updateErrorGroupStates(
        alertableErrors,
        stateMap,
        project.organizationId,
        projectId
      );

      logger.info("[ErrorAlertEvaluator] Evaluation complete", {
        projectId,
        activeErrors: activeErrors.length,
        alertableErrors: alertableErrors.length,
        deliveryJobsEnqueued: alertableErrors.reduce(
          (sum, a) => sum + (channelsByEnvId.get(a.error.environment_id)?.length ?? 0),
          0
        ),
      });
    } catch (error) {
      logger.error("[ErrorAlertEvaluator] Evaluation failed, will retry on next cycle", {
        projectId,
        error,
      });
    } finally {
      await this.selfChain(projectId, nextScheduledAt, minIntervalMs);
    }
  }

  private classifyError(
    error: ActiveErrorsSinceQueryResult,
    state: ErrorGroupState | undefined,
    firstSeenMs: number,
    scheduledAt: number,
    thresholdContext: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
  ): ErrorClassification | null {
    if (!state) {
      return firstSeenMs > scheduledAt ? "new_issue" : null;
    }

    switch (state.status) {
      case "UNRESOLVED":
        return null;

      case "RESOLVED": {
        if (!state.resolvedAt) return null;
        const lastSeenMs = Number(error.last_seen);
        return lastSeenMs > state.resolvedAt.getTime() ? "regression" : null;
      }

      case "IGNORED":
        return this.isIgnoreBreached(state, thresholdContext) ? "unignored" : null;

      default:
        return null;
    }
  }

  private isIgnoreBreached(
    state: ErrorGroupState,
    context: { occurrencesSince: number; windowMs: number; totalOccurrenceCount: number }
  ): boolean {
    if (state.ignoredUntil && state.ignoredUntil.getTime() <= Date.now()) {
      return true;
    }

    if (
      state.ignoredUntilOccurrenceRate !== null &&
      state.ignoredUntilOccurrenceRate !== undefined
    ) {
      const windowMinutes = Math.max(context.windowMs / 60_000, 1);
      const rate = context.occurrencesSince / windowMinutes;
      if (rate > state.ignoredUntilOccurrenceRate) {
        return true;
      }
    }

    if (state.ignoredUntilTotalOccurrences != null && state.ignoredAtOccurrenceCount != null) {
      const occurrencesSinceIgnored =
        context.totalOccurrenceCount - Number(state.ignoredAtOccurrenceCount);
      if (occurrencesSinceIgnored >= state.ignoredUntilTotalOccurrences) {
        return true;
      }
    }

    return false;
  }

  private async resolveChannels(projectId: string): Promise<ProjectAlertChannel[]> {
    return this._replica.projectAlertChannel.findMany({
      where: {
        projectId,
        alertTypes: { has: "ERROR_GROUP" },
        enabled: true,
      },
    });
  }

  private computeMinInterval(channels: ProjectAlertChannel[]): number {
    let min = DEFAULT_INTERVAL_MS;
    for (const ch of channels) {
      const config = ErrorAlertConfig.safeParse(ch.errorAlertConfig);
      if (config.success) {
        min = Math.min(min, config.data.evaluationIntervalMs);
      }
    }
    return min;
  }

  private collectEnvironmentTypes(channels: ProjectAlertChannel[]): RuntimeEnvironmentType[] {
    const types = new Set<RuntimeEnvironmentType>();
    for (const ch of channels) {
      for (const t of ch.environmentTypes) {
        types.add(t);
      }
    }
    return Array.from(types);
  }

  private async resolveEnvironments(
    projectId: string,
    types: RuntimeEnvironmentType[]
  ): Promise<ResolvedEnvironment[]> {
    const envs = await this._replica.runtimeEnvironment.findMany({
      where: {
        projectId,
        type: { in: types },
      },
      select: {
        id: true,
        type: true,
        slug: true,
        branchName: true,
      },
    });

    return envs.map((e) => ({
      id: e.id,
      slug: e.slug,
      type: e.type,
      displayName: e.branchName ?? e.slug,
    }));
  }

  private buildChannelsByEnvId(
    channels: ProjectAlertChannel[],
    environments: ResolvedEnvironment[]
  ): Map<string, ProjectAlertChannel[]> {
    const result = new Map<string, ProjectAlertChannel[]>();
    for (const env of environments) {
      const matching = channels.filter((ch) => ch.environmentTypes.includes(env.type));
      if (matching.length > 0) {
        result.set(env.id, matching);
      }
    }
    return result;
  }

  private async getActiveErrors(
    organizationId: string,
    projectId: string,
    envIds: string[],
    scheduledAt: number
  ): Promise<ActiveErrorsSinceQueryResult[]> {
    const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
      organizationId,
      "query"
    );
    const qb = queryClickhouse.errors.activeErrorsSinceQueryBuilder();
    qb.where("organization_id = {organizationId: String}", { organizationId });
    qb.where("project_id = {projectId: String}", { projectId });
    qb.where("environment_id IN {envIds: Array(String)}", { envIds });
    qb.groupBy("environment_id, task_identifier, error_fingerprint");
    qb.having("toInt64(last_seen) > {scheduledAt: Int64}", {
      scheduledAt,
    });

    const [err, results] = await qb.execute();
    if (err) {
      logger.error("[ErrorAlertEvaluator] Failed to query active errors", { error: err });
      return [];
    }
    return results ?? [];
  }

  private async getErrorGroupStates(
    activeErrors: ActiveErrorsSinceQueryResult[]
  ): Promise<ErrorGroupState[]> {
    if (activeErrors.length === 0) return [];

    return this._replica.errorGroupState.findMany({
      where: {
        OR: activeErrors.map((e) => ({
          environmentId: e.environment_id,
          taskIdentifier: e.task_identifier,
          errorFingerprint: e.error_fingerprint,
        })),
      },
    });
  }

  private buildStateMap(states: ErrorGroupState[]): Map<string, ErrorGroupState> {
    const map = new Map<string, ErrorGroupState>();
    for (const s of states) {
      map.set(`${s.environmentId}:${s.taskIdentifier}:${s.errorFingerprint}`, s);
    }
    return map;
  }

  private async getOccurrenceCountsSince(
    organizationId: string,
    projectId: string,
    envIds: string[],
    scheduledAt: number
  ): Promise<
    Array<{
      environment_id: string;
      task_identifier: string;
      error_fingerprint: string;
      occurrences_since: number;
    }>
  > {
    const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
      organizationId,
      "query"
    );
    const qb = queryClickhouse.errors.occurrenceCountsSinceQueryBuilder();
    qb.where("organization_id = {organizationId: String}", { organizationId });
    qb.where("project_id = {projectId: String}", { projectId });
    qb.where("environment_id IN {envIds: Array(String)}", { envIds });
    qb.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({scheduledAt: Int64}))", {
      scheduledAt,
    });
    qb.groupBy("environment_id, task_identifier, error_fingerprint");

    const [err, results] = await qb.execute();
    if (err) {
      logger.error("[ErrorAlertEvaluator] Failed to query occurrence counts", { error: err });
      return [];
    }
    return results ?? [];
  }

  private buildOccurrenceMap(
    counts: Array<{
      environment_id: string;
      task_identifier: string;
      error_fingerprint: string;
      occurrences_since: number;
    }>
  ): Map<string, number> {
    const map = new Map<string, number>();
    for (const c of counts) {
      map.set(
        `${c.environment_id}:${c.task_identifier}:${c.error_fingerprint}`,
        c.occurrences_since
      );
    }
    return map;
  }

  private async updateErrorGroupStates(
    alertableErrors: AlertableError[],
    stateMap: Map<string, ErrorGroupState>,
    organizationId: string,
    projectId: string
  ): Promise<void> {
    for (const alertable of alertableErrors) {
      const key = `${alertable.error.environment_id}:${alertable.error.task_identifier}:${alertable.error.error_fingerprint}`;
      const state = stateMap.get(key);

      if (state) {
        await this._prisma.errorGroupState.update({
          where: { id: state.id },
          data: {
            status: "UNRESOLVED",
            ignoredUntil: null,
            ignoredUntilOccurrenceRate: null,
            ignoredUntilTotalOccurrences: null,
            ignoredAtOccurrenceCount: null,
            ignoredAt: null,
            ignoredReason: null,
            ignoredByUserId: null,
            resolvedAt: null,
            resolvedInVersion: null,
            resolvedBy: null,
          },
        });
      } else if (alertable.classification === "new_issue") {
        await this._prisma.errorGroupState.create({
          data: {
            organizationId,
            projectId,
            environmentId: alertable.error.environment_id,
            taskIdentifier: alertable.error.task_identifier,
            errorFingerprint: alertable.error.error_fingerprint,
            status: "UNRESOLVED",
          },
        });
      }
    }
  }

  private async selfChain(
    projectId: string,
    nextScheduledAt: number,
    intervalMs: number
  ): Promise<void> {
    await alertsWorker.enqueue({
      id: `evaluateErrorAlerts:${projectId}`,
      job: "v3.evaluateErrorAlerts",
      payload: {
        projectId,
        scheduledAt: nextScheduledAt,
      },
      availableAt: new Date(nextScheduledAt + intervalMs),
    });
  }
}