chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,93 @@
|
||||
// Real PG14 (legacy replica) + PG17 (new) proof for the bulk batch read-through adapter.
|
||||
// We NEVER mock the DB: each closure runs a real `$queryRaw` against the passed container
|
||||
// (crossing the actual PG14↔PG17 boundary) then filters an in-memory seeded set by id —
|
||||
// mirroring readThrough.server.test.ts's `realRead`. The only injected fakes are throwing
|
||||
// spies asserting a store was NEVER touched.
|
||||
import { heteroPostgresTest } from "@internal/testcontainers";
|
||||
import { describe, expect, vi } from "vitest";
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import { hydrateRunsAcrossSeam } from "./BulkActionV2.batchReadThrough.server";
|
||||
|
||||
vi.setConfig({ testTimeout: 60_000 });
|
||||
|
||||
// 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency.
|
||||
const LEGACY_RUN_ID = "run_" + "a".repeat(25);
|
||||
const NEW_RUN_ID = "run_" + "b".repeat(24) + "01";
|
||||
|
||||
type Row = { id: string };
|
||||
|
||||
// Real read against the given container, then return rows for the ids present in `present`.
|
||||
async function realReadFiltered(
|
||||
client: PrismaReplicaClient,
|
||||
ids: string[],
|
||||
present: Set<string>
|
||||
): Promise<Row[]> {
|
||||
await client.$queryRaw<{ marker: number }[]>`SELECT 1 AS marker`;
|
||||
return ids.filter((id) => present.has(id)).map((id) => ({ id }));
|
||||
}
|
||||
|
||||
describe("hydrateRunsAcrossSeam (PG14 legacy replica + PG17 new)", () => {
|
||||
heteroPostgresTest(
|
||||
"(a) mixed page: NEW id from new, LEGACY id from legacy replica; new id never hits legacy",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const onNew = new Set([NEW_RUN_ID]);
|
||||
const onLegacy = new Set([LEGACY_RUN_ID]);
|
||||
|
||||
const readLegacyReplica = vi.fn(
|
||||
async (replica: PrismaReplicaClient, ids: string[]): Promise<Row[]> => {
|
||||
if (ids.includes(NEW_RUN_ID)) {
|
||||
throw new Error("legacy replica must never be probed for a NEW-residency id");
|
||||
}
|
||||
return realReadFiltered(replica, ids, onLegacy);
|
||||
}
|
||||
);
|
||||
|
||||
const rows = await hydrateRunsAcrossSeam<Row>({
|
||||
runIds: [NEW_RUN_ID, LEGACY_RUN_ID],
|
||||
readNew: (client, ids) => realReadFiltered(client, ids, onNew),
|
||||
readLegacyReplica,
|
||||
deps: {
|
||||
splitEnabled: true,
|
||||
newClient: prisma17 as unknown as PrismaReplicaClient,
|
||||
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
|
||||
},
|
||||
});
|
||||
|
||||
const ids = rows.map((r) => r.id).sort();
|
||||
expect(ids).toEqual([LEGACY_RUN_ID, NEW_RUN_ID].sort());
|
||||
expect(readLegacyReplica).toHaveBeenCalledTimes(1);
|
||||
// legacy was only probed for the legacy id
|
||||
expect(readLegacyReplica.mock.calls[0][1]).toEqual([LEGACY_RUN_ID]);
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"(c) passthrough: splitEnabled false reads only the single client; legacy never touched",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const onNew = new Set([NEW_RUN_ID, LEGACY_RUN_ID]);
|
||||
const throwingLegacy = vi.fn(async (): Promise<Row[]> => {
|
||||
throw new Error("readLegacyReplica must never run in single-DB mode");
|
||||
});
|
||||
const readNew = vi.fn((client: PrismaReplicaClient, ids: string[]) =>
|
||||
realReadFiltered(client, ids, onNew)
|
||||
);
|
||||
|
||||
const rows = await hydrateRunsAcrossSeam<Row>({
|
||||
runIds: [NEW_RUN_ID, LEGACY_RUN_ID],
|
||||
readNew,
|
||||
readLegacyReplica: throwingLegacy,
|
||||
deps: {
|
||||
splitEnabled: false,
|
||||
// single collapsed store (use prisma17 here as the "new"/primary analog)
|
||||
newClient: prisma17 as unknown as PrismaReplicaClient,
|
||||
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
|
||||
},
|
||||
});
|
||||
|
||||
const ids = rows.map((r) => r.id).sort();
|
||||
expect(ids).toEqual([LEGACY_RUN_ID, NEW_RUN_ID].sort());
|
||||
expect(readNew).toHaveBeenCalledTimes(1);
|
||||
expect(throwingLegacy).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Batch adapter over the per-id `readThroughRun` (see
|
||||
* `~/v3/runOpsMigration/readThrough.server.ts`). A bulk action processes a PAGE of
|
||||
* member run ids at once, so instead of N per-id round trips this reproduces the
|
||||
* per-id read-through ordering as SET reads:
|
||||
*
|
||||
* 1. single-DB passthrough (splitEnabled === false): ONE read against the collapsed
|
||||
* store, no residency classification, no legacy probe.
|
||||
* 2. split on: classify each id's residency via `ownerEngine`, read NEW for every id
|
||||
* that could be on new (residency NEW *and* legacy-candidates — read-through is
|
||||
* new-FIRST for legacy too), then probe the LEGACY READ REPLICA ONLY for the
|
||||
* legacy-candidates the new read missed.
|
||||
*
|
||||
* Like the per-id layer this NEVER touches a legacy primary/writer — there is no such
|
||||
* handle. An id is read from new OR legacy, never both: legacy is only probed for ids
|
||||
* new missed, so the returned set needs no dedupe.
|
||||
*/
|
||||
import type { PrismaReplicaClient } from "~/db.server";
|
||||
import {
|
||||
runOpsLegacyReplica as defaultLegacyReplica,
|
||||
runOpsNewReplica as defaultNewClient,
|
||||
} from "~/db.server";
|
||||
import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
|
||||
export type SeamReadDeps = {
|
||||
/**
|
||||
* Resolved boot constant. REQUIRED here — the caller resolves it once per
|
||||
* request via `isSplitEnabled()`; this adapter never awaits it itself.
|
||||
*/
|
||||
splitEnabled: boolean;
|
||||
newClient?: PrismaReplicaClient;
|
||||
legacyReplica?: PrismaReplicaClient;
|
||||
logger?: { warn: (m: string, meta?: unknown) => void };
|
||||
};
|
||||
|
||||
type HydrateRunsAcrossSeamInput<T> = {
|
||||
runIds: string[];
|
||||
readNew: (client: PrismaReplicaClient, ids: string[]) => Promise<T[]>;
|
||||
readLegacyReplica: (replica: PrismaReplicaClient, ids: string[]) => Promise<T[]>;
|
||||
deps: SeamReadDeps;
|
||||
};
|
||||
|
||||
/** Every row shape we hydrate carries an `id` (CANCEL select includes it; REPLAY is a full row). */
|
||||
function getId(row: unknown): string {
|
||||
return (row as { id: string }).id;
|
||||
}
|
||||
|
||||
export async function hydrateRunsAcrossSeam<T>(input: HydrateRunsAcrossSeamInput<T>): Promise<T[]> {
|
||||
const { runIds, deps } = input;
|
||||
|
||||
if (runIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const newClient = deps.newClient ?? defaultNewClient;
|
||||
|
||||
// Passthrough: one plain read against the single collapsed store. No residency
|
||||
// classification, no legacy probe, no second connection. When the caller passes its
|
||||
// own `_replica` as `newClient`, this is byte-identical to the pre-migration single-DB read.
|
||||
if (deps.splitEnabled === false) {
|
||||
return input.readNew(newClient, runIds);
|
||||
}
|
||||
|
||||
// Split is on. Classify residency; unclassifiable → LEGACY (probe rather than drop).
|
||||
const newIds: string[] = [];
|
||||
const legacyCandidateIds: string[] = [];
|
||||
for (const runId of runIds) {
|
||||
let residency: "LEGACY" | "NEW";
|
||||
try {
|
||||
residency = ownerEngine(runId);
|
||||
} catch (e) {
|
||||
if (e instanceof UnclassifiableRunId) {
|
||||
deps.logger?.warn("hydrateRunsAcrossSeam: UnclassifiableRunId, treating as LEGACY", {
|
||||
runId,
|
||||
valueLength: e.valueLength,
|
||||
});
|
||||
residency = "LEGACY";
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
if (residency === "NEW") {
|
||||
newIds.push(runId);
|
||||
} else {
|
||||
legacyCandidateIds.push(runId);
|
||||
}
|
||||
}
|
||||
|
||||
// Read NEW for everything that could be on new — NEW-residency ids AND legacy-candidates
|
||||
// (read-through is new-FIRST for legacy too) — in one read.
|
||||
const legacyReplica = deps.legacyReplica ?? defaultLegacyReplica;
|
||||
const newRows = await input.readNew(newClient, [...newIds, ...legacyCandidateIds]);
|
||||
const foundOnNew = new Set(newRows.map(getId));
|
||||
|
||||
// Legacy-candidates the new read missed are probed on the legacy read replica.
|
||||
const legacyToProbe = legacyCandidateIds.filter((id) => !foundOnNew.has(id));
|
||||
|
||||
// Legacy READ REPLICA only — never a legacy writer/primary (no such handle exists).
|
||||
// A member absent from both DBs is simply not hydrated (matching today's `findMany`,
|
||||
// where a missing id yields no row).
|
||||
let legacyRows: T[] = [];
|
||||
if (legacyToProbe.length > 0) {
|
||||
legacyRows = await input.readLegacyReplica(legacyReplica, legacyToProbe);
|
||||
}
|
||||
|
||||
// Order within the page is irrelevant (downstream pMap does not depend on it).
|
||||
return [...newRows, ...legacyRows];
|
||||
}
|
||||
@@ -0,0 +1,616 @@
|
||||
import { BulkActionId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
BulkActionNotificationType,
|
||||
BulkActionStatus,
|
||||
BulkActionType,
|
||||
type PrismaClient,
|
||||
} from "@trigger.dev/database";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import {
|
||||
parseRunListInputOptions,
|
||||
type RunListInputFilters,
|
||||
RunsRepository,
|
||||
} from "~/services/runsRepository/runsRepository.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { ServiceValidationError } from "../common.server";
|
||||
import { commonWorker } from "~/v3/commonWorker.server";
|
||||
import { env } from "~/env.server";
|
||||
import { logger } from "@trigger.dev/sdk";
|
||||
import { CancelTaskRunService } from "../cancelTaskRun.server";
|
||||
import { tryCatch } from "@trigger.dev/core";
|
||||
import { ReplayTaskRunService } from "../replayTaskRun.server";
|
||||
import { WorkerGroupService } from "../worker/workerGroupService.server";
|
||||
import { timeFilters } from "~/components/runs/v3/SharedFilters";
|
||||
import parseDuration from "parse-duration";
|
||||
import { v3BulkActionPath } from "~/utils/pathBuilder";
|
||||
import { formatDateTime } from "~/components/primitives/DateTime";
|
||||
import pMap from "p-map";
|
||||
import { type PrismaReplicaClient } from "~/db.server";
|
||||
import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
|
||||
import { hydrateRunsAcrossSeam, type SeamReadDeps } from "./BulkActionV2.batchReadThrough.server";
|
||||
|
||||
export type CreateBulkActionInput = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
userId?: string | null;
|
||||
action: "cancel" | "replay";
|
||||
filters: RunListInputFilters;
|
||||
title?: string;
|
||||
region?: string;
|
||||
emailNotification?: boolean;
|
||||
triggerSource?: string;
|
||||
};
|
||||
|
||||
export type ProcessToCompletionOptions = {
|
||||
/** Absolute timestamp (ms) after which processing stops and returns incomplete. */
|
||||
deadline?: number;
|
||||
};
|
||||
|
||||
export type ProcessToCompletionResult = {
|
||||
completed: boolean;
|
||||
};
|
||||
|
||||
// How recently a PENDING replay must have made progress to still count against
|
||||
// the per-environment concurrency limit. Every processed batch bumps the
|
||||
// group's `updatedAt`, so a live replay keeps a fresh heartbeat for its whole
|
||||
// life no matter how long it runs, while a replay whose job has exhausted its
|
||||
// retries (and stopped making progress) ages out and frees its slot. This is
|
||||
// wide enough to cover the worst-case gap between batches for a healthy replay
|
||||
// that is retrying.
|
||||
const REPLAY_INFLIGHT_WINDOW_MS = 30 * 60 * 1000;
|
||||
|
||||
export class BulkActionService extends BaseService {
|
||||
#splitEnabledPromise?: Promise<boolean>;
|
||||
|
||||
// Resolves split mode once per service instance and returns the read-through deps for
|
||||
// bulk member hydration. Single-DB: read through the service replica (byte-identical to
|
||||
// the pre-migration read). Split: adapter defaults to run-ops new + legacy read replica.
|
||||
async #seamReadDeps(): Promise<SeamReadDeps> {
|
||||
this.#splitEnabledPromise ??= isSplitEnabled();
|
||||
const splitEnabled = await this.#splitEnabledPromise;
|
||||
return {
|
||||
splitEnabled,
|
||||
newClient: splitEnabled ? undefined : (this._replica as unknown as PrismaReplicaClient),
|
||||
};
|
||||
}
|
||||
|
||||
public async create(input: CreateBulkActionInput) {
|
||||
const { organizationId, projectId, environmentId, userId } = input;
|
||||
const filters = freezeRunListFilters(input.filters);
|
||||
|
||||
// Concurrency guard for replays.
|
||||
// The seek is backed by the (environmentId, status, type) index; the
|
||||
// `updatedAt` window is applied on top so we only count replays that are
|
||||
// actually still making progress. A replay whose job has died stops bumping
|
||||
// `updatedAt` and drops out of the count, so it can't permanently hold a
|
||||
// slot. Aborting a replay (dashboard or API) clears its slot immediately.
|
||||
if (input.action === "replay") {
|
||||
const maxConcurrentReplays = env.BULK_ACTION_MAX_CONCURRENT_REPLAYS;
|
||||
const inFlightReplays = await this._replica.bulkActionGroup.count({
|
||||
where: {
|
||||
environmentId,
|
||||
type: BulkActionType.REPLAY,
|
||||
status: BulkActionStatus.PENDING,
|
||||
updatedAt: { gte: new Date(Date.now() - REPLAY_INFLIGHT_WINDOW_MS) },
|
||||
},
|
||||
});
|
||||
|
||||
if (inFlightReplays >= maxConcurrentReplays) {
|
||||
throw new ServiceValidationError(
|
||||
`You can only run ${maxConcurrentReplays} bulk replays at a time in this environment. Wait for an in-progress replay to finish before starting another.`,
|
||||
429
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Region is a replay-only override that re-routes the replayed runs. It's
|
||||
// stored alongside the run-list filters under a dedicated key so it isn't
|
||||
// mistaken for a `regions` selection filter when the params are parsed.
|
||||
const replayRegion = input.action === "replay" ? input.region : undefined;
|
||||
if (replayRegion) {
|
||||
// Validating the region override up-front so an invalid/unauthorized
|
||||
// region surfaces as a user-input (400) error rather than a 500.
|
||||
const [regionError] = await tryCatch(
|
||||
new WorkerGroupService({ prisma: this._prisma }).getDefaultWorkerGroupForProject({
|
||||
projectId,
|
||||
regionOverride: replayRegion,
|
||||
})
|
||||
);
|
||||
if (regionError) {
|
||||
throw new ServiceValidationError(regionError.message, 400);
|
||||
}
|
||||
}
|
||||
|
||||
const params = {
|
||||
...filters,
|
||||
...(replayRegion ? { replayRegion } : {}),
|
||||
...(input.triggerSource ? { triggerSource: input.triggerSource } : {}),
|
||||
};
|
||||
|
||||
// Count the runs that will be affected by the bulk action
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"standard"
|
||||
);
|
||||
const runsRepository = new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: this._replica as PrismaClient,
|
||||
});
|
||||
const count = await runsRepository.countRuns({
|
||||
organizationId,
|
||||
projectId,
|
||||
environmentId,
|
||||
...filters,
|
||||
});
|
||||
|
||||
// Create the bulk action group
|
||||
const { id, friendlyId } = BulkActionId.generate();
|
||||
const group = await this._prisma.bulkActionGroup.create({
|
||||
data: {
|
||||
id,
|
||||
friendlyId,
|
||||
projectId,
|
||||
environmentId,
|
||||
userId,
|
||||
name: input.title,
|
||||
type: input.action === "cancel" ? BulkActionType.CANCEL : BulkActionType.REPLAY,
|
||||
params,
|
||||
queryName: "bulk_action_v1",
|
||||
totalCount: count,
|
||||
completionNotification:
|
||||
input.emailNotification === true
|
||||
? BulkActionNotificationType.EMAIL
|
||||
: BulkActionNotificationType.NONE,
|
||||
},
|
||||
});
|
||||
|
||||
// Queue the bulk action group for immediate processing
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${group.id}`,
|
||||
job: "processBulkAction",
|
||||
payload: {
|
||||
bulkActionId: group.id,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
bulkActionId: group.friendlyId,
|
||||
};
|
||||
}
|
||||
|
||||
public async processToCompletion(
|
||||
bulkActionId: string,
|
||||
options?: ProcessToCompletionOptions
|
||||
): Promise<ProcessToCompletionResult> {
|
||||
while (true) {
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionId },
|
||||
select: { status: true },
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new Error(`Bulk action group not found: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
return { completed: true };
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
return { completed: false };
|
||||
}
|
||||
|
||||
if (options?.deadline !== undefined && Date.now() >= options.deadline) {
|
||||
return { completed: false };
|
||||
}
|
||||
|
||||
await this.process(bulkActionId, { continueInline: true });
|
||||
}
|
||||
}
|
||||
|
||||
public async process(
|
||||
bulkActionId: string,
|
||||
options?: {
|
||||
continueInline?: boolean;
|
||||
}
|
||||
) {
|
||||
// 1. Get the bulk action group
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionId },
|
||||
select: {
|
||||
status: true,
|
||||
friendlyId: true,
|
||||
projectId: true,
|
||||
environmentId: true,
|
||||
project: {
|
||||
select: {
|
||||
organizationId: true,
|
||||
slug: true,
|
||||
organization: {
|
||||
select: {
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
environment: {
|
||||
select: {
|
||||
slug: true,
|
||||
},
|
||||
},
|
||||
type: true,
|
||||
queryName: true,
|
||||
params: true,
|
||||
cursor: true,
|
||||
completionNotification: true,
|
||||
user: {
|
||||
select: {
|
||||
email: true,
|
||||
},
|
||||
},
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new Error(`Bulk action group not found: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (!group.environmentId || !group.environment) {
|
||||
throw new Error(`Bulk action group has no environment: ${bulkActionId}`);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
logger.log(`Bulk action group already aborted: ${bulkActionId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Parse the params
|
||||
const rawParams = group.params && typeof group.params === "object" ? group.params : {};
|
||||
const finalizeRun = "finalizeRun" in rawParams && (rawParams as any).finalizeRun === true;
|
||||
const replayRegion =
|
||||
"replayRegion" in rawParams && typeof (rawParams as any).replayRegion === "string"
|
||||
? (rawParams as any).replayRegion
|
||||
: undefined;
|
||||
const triggerSource =
|
||||
"triggerSource" in rawParams && typeof (rawParams as any).triggerSource === "string"
|
||||
? (rawParams as any).triggerSource
|
||||
: "dashboard";
|
||||
const filters = parseRunListInputOptions({
|
||||
organizationId: group.project.organizationId,
|
||||
projectId: group.projectId,
|
||||
environmentId: group.environmentId,
|
||||
...rawParams,
|
||||
});
|
||||
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
group.project.organizationId,
|
||||
"standard"
|
||||
);
|
||||
const runsRepository = new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: this._replica as PrismaClient,
|
||||
});
|
||||
|
||||
if (group.queryName !== "bulk_action_v1") {
|
||||
throw new Error(`Bulk action group has invalid query name: ${group.queryName}`);
|
||||
}
|
||||
|
||||
// 2. Get the runs to process in this batch, plus the cursor for the next
|
||||
// batch. The cursor is a composite (created_at, run_id) keyset cursor so the
|
||||
// next batch can't re-include or skip runs.
|
||||
const {
|
||||
runIds: runIdsToProcess,
|
||||
pagination: { nextCursor },
|
||||
} = await runsRepository.listRunIds({
|
||||
...filters,
|
||||
page: {
|
||||
size: env.BULK_ACTION_BATCH_SIZE,
|
||||
cursor:
|
||||
typeof group.cursor === "string" && group.cursor !== null ? group.cursor : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
// 3. Process the runs
|
||||
let successCount = 0;
|
||||
let failureCount = 0;
|
||||
|
||||
switch (group.type) {
|
||||
case BulkActionType.CANCEL: {
|
||||
const cancelService = new CancelTaskRunService(this._prisma);
|
||||
|
||||
const seamDeps = await this.#seamReadDeps();
|
||||
const runs = await hydrateRunsAcrossSeam({
|
||||
runIds: runIdsToProcess,
|
||||
readNew: (client, ids) =>
|
||||
client.taskRun.findMany({
|
||||
where: { id: { in: ids } },
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
friendlyId: true,
|
||||
status: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
taskEventStore: true,
|
||||
},
|
||||
}),
|
||||
readLegacyReplica: (replica, ids) =>
|
||||
replica.taskRun.findMany({
|
||||
where: { id: { in: ids } },
|
||||
select: {
|
||||
id: true,
|
||||
engine: true,
|
||||
friendlyId: true,
|
||||
status: true,
|
||||
createdAt: true,
|
||||
completedAt: true,
|
||||
taskEventStore: true,
|
||||
},
|
||||
}),
|
||||
deps: seamDeps,
|
||||
});
|
||||
|
||||
await pMap(
|
||||
runs,
|
||||
async (run) => {
|
||||
const [error, result] = await tryCatch(
|
||||
cancelService.call(run, {
|
||||
reason: `Bulk action ${group.friendlyId} cancelled run`,
|
||||
bulkActionId: bulkActionId,
|
||||
finalizeRun,
|
||||
})
|
||||
);
|
||||
if (error) {
|
||||
logger.error("Failed to cancel run", {
|
||||
error,
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
if (!result || result.alreadyFinished) {
|
||||
failureCount++;
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
},
|
||||
{ concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
case BulkActionType.REPLAY: {
|
||||
const replayService = new ReplayTaskRunService(this._prisma);
|
||||
|
||||
const seamDeps = await this.#seamReadDeps();
|
||||
const runs = await hydrateRunsAcrossSeam({
|
||||
runIds: runIdsToProcess,
|
||||
readNew: (client, ids) => client.taskRun.findMany({ where: { id: { in: ids } } }),
|
||||
readLegacyReplica: (replica, ids) =>
|
||||
replica.taskRun.findMany({ where: { id: { in: ids } } }),
|
||||
deps: seamDeps,
|
||||
});
|
||||
|
||||
await pMap(
|
||||
runs,
|
||||
async (run) => {
|
||||
const [error, result] = await tryCatch(
|
||||
replayService.call(run, {
|
||||
bulkActionId: bulkActionId,
|
||||
triggerSource,
|
||||
region: replayRegion,
|
||||
})
|
||||
);
|
||||
if (error) {
|
||||
logger.error("Failed to replay run, error", {
|
||||
error,
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
if (!result) {
|
||||
logger.error("Failed to replay run, no result", {
|
||||
runId: run.id,
|
||||
status: run.status,
|
||||
});
|
||||
|
||||
failureCount++;
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
},
|
||||
{ concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A null nextCursor means there is no further page — this batch was the
|
||||
// last (or there were no runs at all), so the action is complete. (An empty
|
||||
// batch also yields a null cursor.)
|
||||
const isFinished = nextCursor === null;
|
||||
|
||||
logger.debug("Bulk action group processed batch", {
|
||||
bulkActionId,
|
||||
organizationId: group.project.organizationId,
|
||||
projectId: group.projectId,
|
||||
environmentId: group.environmentId,
|
||||
batchSize: runIdsToProcess.length,
|
||||
cursor: group.cursor,
|
||||
successCount,
|
||||
failureCount,
|
||||
isFinished,
|
||||
});
|
||||
|
||||
// 4. Update the bulk action group
|
||||
const updatedGroup = await this._prisma.bulkActionGroup.update({
|
||||
where: { id: bulkActionId },
|
||||
data: {
|
||||
// Json column: leave unchanged when there's no next cursor (finished).
|
||||
cursor: nextCursor ?? undefined,
|
||||
successCount: {
|
||||
increment: successCount,
|
||||
},
|
||||
failureCount: {
|
||||
increment: failureCount,
|
||||
},
|
||||
status: isFinished ? BulkActionStatus.COMPLETED : undefined,
|
||||
completedAt: isFinished ? new Date() : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
// 5. If finished, queue a notification and exit
|
||||
if (isFinished) {
|
||||
switch (group.completionNotification) {
|
||||
case BulkActionNotificationType.NONE:
|
||||
return;
|
||||
case BulkActionNotificationType.EMAIL: {
|
||||
if (!group.user) {
|
||||
logger.error("Bulk action group has no user, skipping email notification", {
|
||||
bulkActionId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await commonWorker.enqueue({
|
||||
id: `bulkActionCompletionNotification-${bulkActionId}`,
|
||||
job: "scheduleEmail",
|
||||
payload: {
|
||||
to: group.user.email,
|
||||
email: "bulk-action-completed",
|
||||
bulkActionId: group.friendlyId,
|
||||
url: `${env.LOGIN_ORIGIN}${v3BulkActionPath(
|
||||
{
|
||||
slug: group.project.organization.slug,
|
||||
},
|
||||
{
|
||||
slug: group.project.slug,
|
||||
},
|
||||
{
|
||||
slug: group.environment.slug,
|
||||
},
|
||||
{
|
||||
friendlyId: group.friendlyId,
|
||||
}
|
||||
)}`,
|
||||
totalCount: updatedGroup.totalCount,
|
||||
successCount: updatedGroup.successCount,
|
||||
failureCount: updatedGroup.failureCount,
|
||||
type: group.type,
|
||||
createdAt: formatDateTime(group.createdAt, "UTC", [], true, true),
|
||||
completedAt: formatDateTime(group.completedAt ?? new Date(), "UTC", [], true, true),
|
||||
},
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// 6. If there are more runs to process, queue the next batch
|
||||
if (options?.continueInline) {
|
||||
return;
|
||||
}
|
||||
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${bulkActionId}`,
|
||||
job: "processBulkAction",
|
||||
payload: { bulkActionId },
|
||||
availableAt: new Date(Date.now() + env.BULK_ACTION_BATCH_DELAY_MS),
|
||||
});
|
||||
}
|
||||
|
||||
public async abort(friendlyId: string, environmentId: string) {
|
||||
const group = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { friendlyId, environmentId },
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!group) {
|
||||
throw new ServiceValidationError(`Bulk action not found: ${friendlyId}`, 404);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.COMPLETED) {
|
||||
throw new ServiceValidationError(`Bulk action group already completed: ${friendlyId}`, 409);
|
||||
}
|
||||
|
||||
if (group.status === BulkActionStatus.ABORTED) {
|
||||
throw new ServiceValidationError(`Bulk action group already aborted: ${friendlyId}`, 409);
|
||||
}
|
||||
|
||||
//ack the job (this doesn't guarantee it won't run again)
|
||||
await commonWorker.ack(`processBulkAction-${group.id}`);
|
||||
|
||||
await this._prisma.bulkActionGroup.update({
|
||||
where: { id: group.id },
|
||||
data: { status: BulkActionStatus.ABORTED },
|
||||
});
|
||||
|
||||
return {
|
||||
bulkActionId: friendlyId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function freezeRunListFilters(filters: RunListInputFilters): RunListInputFilters {
|
||||
const {
|
||||
cursor: _cursor,
|
||||
direction: _direction,
|
||||
...frozenFilters
|
||||
} = filters as RunListInputFilters & {
|
||||
cursor?: string;
|
||||
direction?: "forward" | "backward";
|
||||
};
|
||||
|
||||
// Explicit run-id selections target specific, already-existing runs, so we
|
||||
// don't apply a time bound (which could otherwise exclude a selected run).
|
||||
if (frozenFilters.runId?.length) {
|
||||
return frozenFilters;
|
||||
}
|
||||
|
||||
const { period } = timeFilters({
|
||||
period: frozenFilters.period,
|
||||
from: frozenFilters.from,
|
||||
to: frozenFilters.to,
|
||||
});
|
||||
|
||||
// We fix the time period to a from/to date
|
||||
if (period) {
|
||||
const periodMs = parseDuration(period);
|
||||
if (!periodMs) {
|
||||
throw new Error(`Invalid period: ${period}`);
|
||||
}
|
||||
|
||||
const to = new Date();
|
||||
const from = new Date(to.getTime() - periodMs);
|
||||
frozenFilters.from = from.getTime();
|
||||
frozenFilters.to = to.getTime();
|
||||
frozenFilters.period = undefined;
|
||||
return frozenFilters;
|
||||
}
|
||||
|
||||
// If no to date is set, we lock it to now
|
||||
if (!frozenFilters.to) {
|
||||
frozenFilters.to = Date.now();
|
||||
}
|
||||
|
||||
frozenFilters.period = undefined;
|
||||
|
||||
return frozenFilters;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { type BulkActionType } from "@trigger.dev/database";
|
||||
import { bulkActionVerb } from "~/components/runs/v3/BulkAction";
|
||||
import { BULK_ACTION_RUN_LIMIT } from "~/consts";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../../friendlyIdentifiers";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { PerformBulkActionService } from "./performBulkAction.server";
|
||||
|
||||
type BulkAction = {
|
||||
projectId: string;
|
||||
action: BulkActionType;
|
||||
runIds: string[];
|
||||
};
|
||||
|
||||
export class CreateBulkActionService extends BaseService {
|
||||
public async call({ projectId, action, runIds }: BulkAction) {
|
||||
const group = await this._prisma.bulkActionGroup.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("bulk"),
|
||||
projectId,
|
||||
type: action,
|
||||
},
|
||||
});
|
||||
|
||||
//limit to the first X runs
|
||||
const passedTooManyRuns = runIds.length > BULK_ACTION_RUN_LIMIT;
|
||||
runIds = runIds.slice(0, BULK_ACTION_RUN_LIMIT);
|
||||
|
||||
const _items = await this._prisma.bulkActionItem.createMany({
|
||||
data: runIds.map((runId) => ({
|
||||
friendlyId: generateFriendlyId("bulkitem"),
|
||||
type: action,
|
||||
groupId: group.id,
|
||||
sourceRunId: runId,
|
||||
})),
|
||||
});
|
||||
|
||||
logger.debug("Created bulk action group", {
|
||||
groupId: group.id,
|
||||
action,
|
||||
runIds,
|
||||
});
|
||||
|
||||
await PerformBulkActionService.enqueue(group.id, this._prisma);
|
||||
|
||||
let message = bulkActionVerb(action);
|
||||
if (passedTooManyRuns) {
|
||||
message += ` the first ${BULK_ACTION_RUN_LIMIT} runs`;
|
||||
} else {
|
||||
message += ` ${runIds.length} runs`;
|
||||
}
|
||||
|
||||
return {
|
||||
id: group.id,
|
||||
friendlyId: group.friendlyId,
|
||||
runCount: runIds.length,
|
||||
message,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import assertNever from "assert-never";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
import { BaseService } from "../baseService.server";
|
||||
import { CancelTaskRunService } from "../cancelTaskRun.server";
|
||||
import { ReplayTaskRunService } from "../replayTaskRun.server";
|
||||
|
||||
export class PerformBulkActionService extends BaseService {
|
||||
public async performBulkActionItem(bulkActionItemId: string) {
|
||||
const item = await this._prisma.bulkActionItem.findFirst({
|
||||
where: { id: bulkActionItemId },
|
||||
include: {
|
||||
sourceRun: true,
|
||||
destinationRun: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!item) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (item.status !== "PENDING") {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (item.type) {
|
||||
case "REPLAY": {
|
||||
const service = new ReplayTaskRunService(this._prisma);
|
||||
const result = await service.call(item.sourceRun, { triggerSource: "dashboard" });
|
||||
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
destinationRunId: result?.id,
|
||||
status: result ? "COMPLETED" : "FAILED",
|
||||
error: result ? undefined : "Failed to replay task run",
|
||||
},
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "CANCEL": {
|
||||
const service = new CancelTaskRunService(this._prisma);
|
||||
|
||||
const result = await service.call(item.sourceRun);
|
||||
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
destinationRunId: item.sourceRun.id,
|
||||
status: result ? "COMPLETED" : "FAILED",
|
||||
error: result ? undefined : "Task wasn't cancelable",
|
||||
},
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
assertNever(item.type);
|
||||
}
|
||||
}
|
||||
|
||||
const groupItems = await this._prisma.bulkActionItem.findMany({
|
||||
where: { groupId: item.groupId },
|
||||
select: {
|
||||
status: true,
|
||||
},
|
||||
});
|
||||
|
||||
const isGroupCompleted = groupItems.every((item) => item.status !== "PENDING");
|
||||
|
||||
if (isGroupCompleted) {
|
||||
await this._prisma.bulkActionItem.update({
|
||||
where: { id: item.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async enqueueBulkActionItem(bulkActionItemId: string, groupId: string) {
|
||||
await workerQueue.enqueue(
|
||||
"v3.performBulkActionItem",
|
||||
{
|
||||
bulkActionItemId,
|
||||
},
|
||||
{
|
||||
jobKey: `performBulkActionItem:${bulkActionItemId}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public async call(bulkActionGroupId: string) {
|
||||
const actionGroup = await this._prisma.bulkActionGroup.findFirst({
|
||||
where: { id: bulkActionGroupId },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
if (!actionGroup) {
|
||||
return;
|
||||
}
|
||||
|
||||
const items = await this._prisma.bulkActionItem.findMany({
|
||||
where: { groupId: bulkActionGroupId },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
for (const item of items) {
|
||||
await this.enqueueBulkActionItem(item.id, bulkActionGroupId);
|
||||
}
|
||||
}
|
||||
|
||||
static async enqueue(bulkActionGroupId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
|
||||
return await workerQueue.enqueue(
|
||||
"v3.performBulkAction",
|
||||
{
|
||||
bulkActionGroupId,
|
||||
},
|
||||
{
|
||||
tx,
|
||||
runAt,
|
||||
jobKey: `performBulkAction:${bulkActionGroupId}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user