chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,520 @@
|
||||
import { type ClickhouseQueryBuilder } from "@internal/clickhouse";
|
||||
import { ErrorId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
type FilterRunsOptions,
|
||||
type IRunsRepository,
|
||||
type ListRunsOptions,
|
||||
type RunIdsPage,
|
||||
type RunListInputOptions,
|
||||
type RunsRepositoryOptions,
|
||||
type TagListOptions,
|
||||
convertRunListInputOptionsToFilterRunsOptions,
|
||||
} from "./runsRepository.server";
|
||||
import parseDuration from "parse-duration";
|
||||
import { decodeRunsCursor, encodeRunsCursor } from "./runsCursor.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
import { type PrismaClientOrTransaction } from "~/db.server";
|
||||
|
||||
type RunCursorRow = { runId: string; createdAt: number };
|
||||
|
||||
/**
|
||||
* Hydrates a set of rows for a ClickHouse-derived run-id set against the given
|
||||
* read client. The closure MUST select `id` so `#hydrateRunsByIds` can key
|
||||
* set-membership and re-impose ordering; the call site projects `id` away if its
|
||||
* result type excludes it.
|
||||
*/
|
||||
type HydrateFn<T extends { id: string }> = (
|
||||
client: PrismaClientOrTransaction,
|
||||
ids: string[]
|
||||
) => Promise<T[]>;
|
||||
|
||||
export class ClickHouseRunsRepository implements IRunsRepository {
|
||||
constructor(private readonly options: RunsRepositoryOptions) {}
|
||||
|
||||
get name() {
|
||||
return "clickhouse";
|
||||
}
|
||||
|
||||
async runExistsInEnvironment(options: {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
createdAtLowerBoundMs?: number;
|
||||
}): Promise<boolean> {
|
||||
const queryBuilder = this.options.clickhouse.taskRuns.existsQueryBuilder();
|
||||
|
||||
queryBuilder
|
||||
.where("organization_id = {organizationId: String}", {
|
||||
organizationId: options.organizationId,
|
||||
})
|
||||
.where("project_id = {projectId: String}", { projectId: options.projectId })
|
||||
.where("environment_id = {environmentId: String}", { environmentId: options.environmentId });
|
||||
|
||||
if (typeof options.createdAtLowerBoundMs === "number") {
|
||||
queryBuilder.where("created_at >= fromUnixTimestamp64Milli({createdAtLowerBound: Int64})", {
|
||||
createdAtLowerBound: options.createdAtLowerBoundMs,
|
||||
});
|
||||
}
|
||||
|
||||
queryBuilder.limit(1);
|
||||
|
||||
const [queryError, result] = await queryBuilder.execute();
|
||||
|
||||
if (queryError) {
|
||||
throw queryError;
|
||||
}
|
||||
|
||||
return (result?.length ?? 0) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the keyset-paginated query and returns `{ runId, createdAt }` rows
|
||||
* (one extra beyond `page.size` to signal "has more"). The ordering is always
|
||||
* the composite `(created_at, run_id)`; the cursor predicate must match it.
|
||||
*
|
||||
* Composite cursors carry both components, so we cut on the
|
||||
* `(created_at, run_id)` tuple — sound regardless of how run_id order relates
|
||||
* to created_at order. Legacy bare-run_id cursors fall back to the old
|
||||
* `run_id`-only predicate (knowingly unsound) for backwards compatibility
|
||||
* with in-flight cursors.
|
||||
*/
|
||||
private async listRunRows(options: ListRunsOptions): Promise<RunCursorRow[]> {
|
||||
const queryBuilder = this.options.clickhouse.taskRuns.queryBuilder();
|
||||
applyRunFiltersToQueryBuilder(
|
||||
queryBuilder,
|
||||
await convertRunListInputOptionsToFilterRunsOptions(
|
||||
options,
|
||||
this.options.prisma,
|
||||
this.options.runStore ?? runStore
|
||||
)
|
||||
);
|
||||
|
||||
const forward = options.page.direction === "forward" || !options.page.direction;
|
||||
|
||||
if (options.page.cursor) {
|
||||
const decoded = decodeRunsCursor(options.page.cursor);
|
||||
|
||||
if (forward) {
|
||||
if (decoded.kind === "composite") {
|
||||
queryBuilder.where(
|
||||
"(created_at, run_id) < (fromUnixTimestamp64Milli({cursorCreatedAt: Int64}), {runId: String})",
|
||||
{ cursorCreatedAt: decoded.createdAt, runId: decoded.runId }
|
||||
);
|
||||
} else {
|
||||
queryBuilder.where("run_id < {runId: String}", { runId: decoded.runId });
|
||||
}
|
||||
queryBuilder.orderBy("created_at DESC, run_id DESC");
|
||||
} else {
|
||||
if (decoded.kind === "composite") {
|
||||
queryBuilder.where(
|
||||
"(created_at, run_id) > (fromUnixTimestamp64Milli({cursorCreatedAt: Int64}), {runId: String})",
|
||||
{ cursorCreatedAt: decoded.createdAt, runId: decoded.runId }
|
||||
);
|
||||
} else {
|
||||
queryBuilder.where("run_id > {runId: String}", { runId: decoded.runId });
|
||||
}
|
||||
queryBuilder.orderBy("created_at ASC, run_id ASC");
|
||||
}
|
||||
|
||||
queryBuilder.limit(options.page.size + 1);
|
||||
} else {
|
||||
// Initial page - no cursor provided
|
||||
queryBuilder.orderBy("created_at DESC, run_id DESC").limit(options.page.size + 1);
|
||||
}
|
||||
|
||||
const [queryError, result] = await queryBuilder.execute();
|
||||
|
||||
if (queryError) {
|
||||
throw queryError;
|
||||
}
|
||||
|
||||
return result.map((row) => ({ runId: row.run_id, createdAt: row.created_at_ms }));
|
||||
}
|
||||
|
||||
/**
|
||||
* A keyset-paginated page of run ids ordered by `(created_at, run_id)`, plus
|
||||
* the cursors to page forward/backward. Cursors are composite tokens that
|
||||
* match the ordering, so pagination can't duplicate or skip runs even when
|
||||
* run_id order diverges from created_at order. This is the single source of
|
||||
* cursor construction — `listRuns` and bulk actions both build on it.
|
||||
*/
|
||||
async listRunIds(options: ListRunsOptions): Promise<RunIdsPage> {
|
||||
const rows = await this.listRunRows(options);
|
||||
|
||||
// listRunRows fetches one extra row beyond page.size to detect "has more".
|
||||
const hasMore = rows.length > options.page.size;
|
||||
|
||||
const cursorFor = (row: RunCursorRow | undefined): string | null =>
|
||||
row ? encodeRunsCursor(row.createdAt, row.runId) : null;
|
||||
|
||||
let nextCursor: string | null = null;
|
||||
let previousCursor: string | null = null;
|
||||
|
||||
const direction = options.page.direction ?? "forward";
|
||||
switch (direction) {
|
||||
case "forward": {
|
||||
previousCursor = options.page.cursor ? cursorFor(rows.at(0)) : null;
|
||||
if (hasMore) {
|
||||
// The next cursor is the last run on this page.
|
||||
nextCursor = cursorFor(rows[options.page.size - 1]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "backward": {
|
||||
const reversedRows = [...rows].reverse();
|
||||
if (hasMore) {
|
||||
previousCursor = cursorFor(reversedRows.at(1));
|
||||
nextCursor = cursorFor(reversedRows.at(options.page.size));
|
||||
} else {
|
||||
// No newer rows, so there's no previous (newer) page. The next
|
||||
// (older) cursor is the oldest row on this page = rows[0] (rows are
|
||||
// ASC here). Index by the actual row count, not page.size — on a
|
||||
// partial page (fewer than page.size rows) page.size-1 overshoots
|
||||
// and would null the cursor, stranding forward navigation.
|
||||
nextCursor = cursorFor(rows.at(0));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// The page is always the first `page.size` rows of the result. listRunRows
|
||||
// fetches one extra row only to detect `hasMore`; that extra row is the
|
||||
// farthest from the cursor in BOTH directions (forward orders DESC, backward
|
||||
// orders ASC), so it's always the trailing element to drop — never the
|
||||
// leading one. (Slicing `[1, size+1]` for backward dropped the row closest
|
||||
// to the cursor and kept the has-more sentinel, straddling two pages.)
|
||||
const runIds = rows.slice(0, options.page.size).map((row) => row.runId);
|
||||
|
||||
return { runIds, pagination: { nextCursor, previousCursor } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Hydrates a ClickHouse-derived run-id set from the run-ops store.
|
||||
* Split ON: new run-ops client first, then the LEGACY RUN-OPS READ REPLICA ONLY
|
||||
* for ids not known-migrated — never the legacy primary. The mixed-residency
|
||||
* fan-out lives here because `RoutingRunStore.findRuns` punts it.
|
||||
* Split OFF (single-DB / self-host): one plain `store.findRuns(args, prisma)`
|
||||
* (passthrough) — no legacy read, no known-migrated probe, no second connection.
|
||||
*/
|
||||
async #hydrateRunsByIds<T extends { id: string }>(
|
||||
runIds: string[],
|
||||
hydrate: HydrateFn<T>
|
||||
): Promise<T[]> {
|
||||
if (runIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const splitEnabled = this.options.readThrough?.splitEnabled ?? false;
|
||||
|
||||
let rows: T[];
|
||||
if (!splitEnabled) {
|
||||
rows = await hydrate(this.options.prisma, runIds);
|
||||
} else {
|
||||
const newClient = this.options.readThrough?.newClient ?? this.options.prisma;
|
||||
const legacyReplica = this.options.readThrough?.legacyReplica ?? this.options.prisma;
|
||||
|
||||
const newRows = await hydrate(newClient, runIds);
|
||||
const foundIds = new Set(newRows.map((r) => r.id));
|
||||
const missing = runIds.filter((id) => !foundIds.has(id));
|
||||
|
||||
// Any id not hydrated from the new store is probed on the legacy replica.
|
||||
const toProbeLegacy = missing;
|
||||
|
||||
const legacyRows = toProbeLegacy.length ? await hydrate(legacyReplica, toProbeLegacy) : [];
|
||||
rows = [...newRows, ...legacyRows];
|
||||
}
|
||||
|
||||
// Preserve the ClickHouse keyset order (created_at desc, run_id desc) by re-ordering the
|
||||
// hydrated rows to match the input `runIds`. Sorting by raw `id` was only ~chronological
|
||||
// when every id was a time-prefixed cuid; a mixed cuid/run-ops id page sorts the two id-spaces
|
||||
// into separate blocks, burying recent runs. Rows whose PG row is gone (e.g. past
|
||||
// retention) drop out, exactly as before.
|
||||
const byId = new Map(rows.map((r) => [r.id, r] as const));
|
||||
return runIds.map((id) => byId.get(id)).filter((r): r is T => r !== undefined);
|
||||
}
|
||||
|
||||
async listFriendlyRunIds(options: ListRunsOptions) {
|
||||
// First get internal IDs from ClickHouse
|
||||
const { runIds } = await this.listRunIds(options);
|
||||
|
||||
if (runIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const store = this.options.runStore ?? runStore;
|
||||
|
||||
// Then get friendly IDs from the run-ops store (id added for set-membership;
|
||||
// projected away below so the returned shape stays `string[]`).
|
||||
const runs = await this.#hydrateRunsByIds(runIds, (client, ids) =>
|
||||
store.findRuns(
|
||||
{
|
||||
where: { id: { in: ids } },
|
||||
select: { id: true, friendlyId: true },
|
||||
},
|
||||
client
|
||||
)
|
||||
);
|
||||
|
||||
return runs.map((run) => run.friendlyId);
|
||||
}
|
||||
|
||||
async listRuns(options: ListRunsOptions) {
|
||||
const { runIds, pagination } = await this.listRunIds(options);
|
||||
|
||||
const store = this.options.runStore ?? runStore;
|
||||
|
||||
let runs = await this.#hydrateRunsByIds(runIds, (client, ids) =>
|
||||
store.findRuns(
|
||||
{
|
||||
where: {
|
||||
id: {
|
||||
in: ids,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
id: "desc",
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
friendlyId: true,
|
||||
taskIdentifier: true,
|
||||
taskVersion: true,
|
||||
runtimeEnvironmentId: true,
|
||||
status: true,
|
||||
createdAt: true,
|
||||
startedAt: true,
|
||||
lockedAt: true,
|
||||
delayUntil: true,
|
||||
updatedAt: true,
|
||||
completedAt: true,
|
||||
isTest: true,
|
||||
spanId: true,
|
||||
idempotencyKey: true,
|
||||
ttl: true,
|
||||
expiredAt: true,
|
||||
costInCents: true,
|
||||
baseCostInCents: true,
|
||||
usageDurationMs: true,
|
||||
runTags: true,
|
||||
depth: true,
|
||||
rootTaskRunId: true,
|
||||
batchId: true,
|
||||
metadata: true,
|
||||
metadataType: true,
|
||||
machinePreset: true,
|
||||
queue: true,
|
||||
workerQueue: true,
|
||||
region: true,
|
||||
annotations: true,
|
||||
},
|
||||
},
|
||||
client
|
||||
)
|
||||
);
|
||||
|
||||
// ClickHouse is slightly delayed, so we're going to do in-memory status filtering too
|
||||
if (options.statuses && options.statuses.length > 0) {
|
||||
runs = runs.filter((run) => options.statuses!.includes(run.status));
|
||||
}
|
||||
|
||||
return {
|
||||
runs,
|
||||
pagination,
|
||||
};
|
||||
}
|
||||
|
||||
async countRuns(options: RunListInputOptions) {
|
||||
const queryBuilder = this.options.clickhouse.taskRuns.countQueryBuilder();
|
||||
applyRunFiltersToQueryBuilder(
|
||||
queryBuilder,
|
||||
await convertRunListInputOptionsToFilterRunsOptions(
|
||||
options,
|
||||
this.options.prisma,
|
||||
this.options.runStore ?? runStore
|
||||
)
|
||||
);
|
||||
|
||||
const [queryError, result] = await queryBuilder.execute();
|
||||
|
||||
if (queryError) {
|
||||
throw queryError;
|
||||
}
|
||||
|
||||
if (result.length === 0) {
|
||||
throw new Error("No count rows returned");
|
||||
}
|
||||
|
||||
return result[0].count;
|
||||
}
|
||||
|
||||
async listTags(options: TagListOptions) {
|
||||
const queryBuilder = this.options.clickhouse.taskRuns
|
||||
.tagQueryBuilder()
|
||||
.where("organization_id = {organizationId: String}", {
|
||||
organizationId: options.organizationId,
|
||||
})
|
||||
.where("project_id = {projectId: String}", {
|
||||
projectId: options.projectId,
|
||||
})
|
||||
.where("environment_id = {environmentId: String}", {
|
||||
environmentId: options.environmentId,
|
||||
});
|
||||
|
||||
const periodMs = options.period ? (parseDuration(options.period) ?? undefined) : undefined;
|
||||
if (periodMs) {
|
||||
queryBuilder.where("created_at >= fromUnixTimestamp64Milli({period: Int64})", {
|
||||
period: new Date(Date.now() - periodMs).getTime(),
|
||||
});
|
||||
}
|
||||
|
||||
if (options.from) {
|
||||
queryBuilder.where("created_at >= fromUnixTimestamp64Milli({from: Int64})", {
|
||||
from: options.from,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.to) {
|
||||
queryBuilder.where("created_at <= fromUnixTimestamp64Milli({to: Int64})", { to: options.to });
|
||||
}
|
||||
|
||||
// Filter by query (case-insensitive contains search)
|
||||
if (options.query && options.query.trim().length > 0) {
|
||||
queryBuilder.where("positionCaseInsensitiveUTF8(tag, {query: String}) > 0", {
|
||||
query: options.query,
|
||||
});
|
||||
}
|
||||
|
||||
// Add ordering and pagination
|
||||
queryBuilder.orderBy("tag ASC").limit(options.limit);
|
||||
|
||||
const [queryError, result] = await queryBuilder.execute();
|
||||
|
||||
if (queryError) {
|
||||
throw queryError;
|
||||
}
|
||||
|
||||
return {
|
||||
tags: result.map((row) => row.tag),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function applyRunFiltersToQueryBuilder<T>(
|
||||
queryBuilder: ClickhouseQueryBuilder<T>,
|
||||
options: FilterRunsOptions
|
||||
) {
|
||||
queryBuilder
|
||||
.where("organization_id = {organizationId: String}", {
|
||||
organizationId: options.organizationId,
|
||||
})
|
||||
.where("project_id = {projectId: String}", {
|
||||
projectId: options.projectId,
|
||||
})
|
||||
.where("environment_id = {environmentId: String}", {
|
||||
environmentId: options.environmentId,
|
||||
});
|
||||
|
||||
if (options.tasks && options.tasks.length > 0) {
|
||||
queryBuilder.where("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks });
|
||||
}
|
||||
|
||||
if (options.versions && options.versions.length > 0) {
|
||||
queryBuilder.where("task_version IN {versions: Array(String)}", {
|
||||
versions: options.versions,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.statuses && options.statuses.length > 0) {
|
||||
queryBuilder.where("status IN {statuses: Array(String)}", { statuses: options.statuses });
|
||||
}
|
||||
|
||||
if (options.tags && options.tags.length > 0) {
|
||||
// Both hasAny and hasAll are served by the tags bloom_filter skip index.
|
||||
const tagsFn = options.tagsMatch === "all" ? "hasAll" : "hasAny";
|
||||
queryBuilder.where(`${tagsFn}(tags, {tags: Array(String)})`, { tags: options.tags });
|
||||
}
|
||||
|
||||
if (options.scheduleId) {
|
||||
queryBuilder.where("schedule_id = {scheduleId: String}", { scheduleId: options.scheduleId });
|
||||
}
|
||||
|
||||
// Period is a number of milliseconds duration
|
||||
if (options.period) {
|
||||
queryBuilder.where("created_at >= fromUnixTimestamp64Milli({period: Int64})", {
|
||||
period: new Date(Date.now() - options.period).getTime(),
|
||||
});
|
||||
}
|
||||
|
||||
if (options.from) {
|
||||
queryBuilder.where("created_at >= fromUnixTimestamp64Milli({from: Int64})", {
|
||||
from: options.from,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.to) {
|
||||
queryBuilder.where("created_at <= fromUnixTimestamp64Milli({to: Int64})", { to: options.to });
|
||||
}
|
||||
|
||||
if (typeof options.isTest === "boolean") {
|
||||
queryBuilder.where("is_test = {isTest: Boolean}", { isTest: options.isTest });
|
||||
}
|
||||
|
||||
if (options.rootOnly) {
|
||||
queryBuilder.where("root_run_id = ''");
|
||||
}
|
||||
|
||||
if (options.batchId) {
|
||||
queryBuilder.where("batch_id = {batchId: String}", { batchId: options.batchId });
|
||||
}
|
||||
|
||||
if (options.bulkId) {
|
||||
queryBuilder.where("hasAny(bulk_action_group_ids, {bulkActionGroupIds: Array(String)})", {
|
||||
bulkActionGroupIds: [options.bulkId],
|
||||
});
|
||||
}
|
||||
|
||||
if (options.runId && options.runId.length > 0) {
|
||||
// it's important that in the query it's "runIds", otherwise it clashes with the cursor which is called "runId"
|
||||
queryBuilder.where("friendly_id IN {runIds: Array(String)}", {
|
||||
runIds: options.runId.map((runId) => RunId.toFriendlyId(runId)),
|
||||
});
|
||||
}
|
||||
|
||||
if (options.queues && options.queues.length > 0) {
|
||||
queryBuilder.where("queue IN {queues: Array(String)}", { queues: options.queues });
|
||||
}
|
||||
|
||||
if (options.regions && options.regions.length > 0) {
|
||||
queryBuilder.where("if(region != '', region, worker_queue) IN {regions: Array(String)}", {
|
||||
regions: options.regions,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.machines && options.machines.length > 0) {
|
||||
queryBuilder.where("machine_preset IN {machines: Array(String)}", {
|
||||
machines: options.machines,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.errorId) {
|
||||
queryBuilder.where("error_fingerprint = {errorFingerprint: String}", {
|
||||
errorFingerprint: ErrorId.toId(options.errorId),
|
||||
});
|
||||
}
|
||||
|
||||
if (options.taskKinds && options.taskKinds.length > 0) {
|
||||
const includesStandard = options.taskKinds.includes("STANDARD");
|
||||
// Include empty string when filtering for STANDARD (default value for pre-existing runs)
|
||||
const effectiveKinds = includesStandard ? [...options.taskKinds, ""] : options.taskKinds;
|
||||
|
||||
if (effectiveKinds.length === 1) {
|
||||
queryBuilder.where("task_kind = {taskKind: String}", {
|
||||
taskKind: effectiveKinds[0]!,
|
||||
});
|
||||
} else {
|
||||
queryBuilder.where("task_kind IN {taskKinds: Array(String)}", {
|
||||
taskKinds: effectiveKinds,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Cursor encoding for keyset pagination over `(created_at, run_id)`.
|
||||
*
|
||||
* The list query orders by the composite key `(created_at, run_id)`, so a sound
|
||||
* cursor must carry BOTH components — cutting on `run_id` alone re-includes and
|
||||
* skips rows whenever `run_id` order diverges from `created_at` order.
|
||||
*
|
||||
* A cursor is an opaque URL-safe base64 token wrapping `{ c: createdAtMs, r:
|
||||
* runId }`. Cursors are server-issued (the SDK just echoes
|
||||
* `pagination.next`/`previous` back), so this format needs no client update.
|
||||
*
|
||||
* Legacy cursors were the bare internal run_id (a cuid). They are detected by
|
||||
* decode failure: a cuid base64-decodes to non-JSON bytes, so it falls through
|
||||
* to `{ kind: "legacy" }` and the old (knowingly unsound) `run_id`-only
|
||||
* predicate. In-flight legacy cursors keep working and drain naturally.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
export type DecodedRunsCursor =
|
||||
| { kind: "composite"; createdAt: number; runId: string }
|
||||
| { kind: "legacy"; runId: string };
|
||||
|
||||
// `c` = created_at (ms since epoch), `r` = run_id. Short keys keep the token small.
|
||||
const CompositeCursor = z.object({
|
||||
c: z.number().int(),
|
||||
r: z.string().min(1),
|
||||
});
|
||||
|
||||
export function encodeRunsCursor(createdAtMs: number, runId: string): string {
|
||||
return Buffer.from(JSON.stringify({ c: createdAtMs, r: runId })).toString("base64url");
|
||||
}
|
||||
|
||||
export function decodeRunsCursor(cursor: string): DecodedRunsCursor {
|
||||
try {
|
||||
const parsed = CompositeCursor.safeParse(
|
||||
JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"))
|
||||
);
|
||||
if (parsed.success) {
|
||||
return { kind: "composite", createdAt: parsed.data.c, runId: parsed.data.r };
|
||||
}
|
||||
} catch {
|
||||
// JSON.parse threw — not a composite cursor.
|
||||
}
|
||||
|
||||
return { kind: "legacy", runId: cursor };
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
import { type ClickHouse } from "@internal/clickhouse";
|
||||
import { type RunStore } from "@internal/run-store";
|
||||
import { type Tracer } from "@internal/tracing";
|
||||
import { type Logger, type LogLevel } from "@trigger.dev/core/logger";
|
||||
import { MachinePresetName } from "@trigger.dev/core/v3";
|
||||
import { BulkActionId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type Prisma, TaskRunStatus } from "@trigger.dev/database";
|
||||
import parseDuration from "parse-duration";
|
||||
import { z } from "zod";
|
||||
import { timeFilters } from "~/components/runs/v3/SharedFilters";
|
||||
import { type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { runStore as defaultRunStore } from "~/v3/runStore.server";
|
||||
import { startActiveSpan } from "~/v3/tracer.server";
|
||||
import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server";
|
||||
|
||||
export type RunsRepositoryOptions = {
|
||||
clickhouse: ClickHouse;
|
||||
prisma: PrismaClientOrTransaction;
|
||||
logger?: Logger;
|
||||
logLevel?: LogLevel;
|
||||
tracer?: Tracer;
|
||||
|
||||
// Injectable run-ops store; defaults to the `~/v3/runStore.server` singleton
|
||||
// (passthrough). The list-hydrate fan-out below does not depend on the store
|
||||
// routing mixed-residency id sets — it applies the read-through fan-out itself.
|
||||
runStore?: RunStore;
|
||||
|
||||
// Run-ops read-through wiring for the list hydrate. Omitted => passthrough.
|
||||
readThrough?: {
|
||||
// `legacyReplica` is a READ REPLICA handle only — there is no legacy-primary field.
|
||||
newClient?: PrismaClientOrTransaction;
|
||||
legacyReplica?: PrismaClientOrTransaction;
|
||||
// Resolved boot constant; when false the split branch is never entered.
|
||||
splitEnabled?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
const RunStatus = z.enum(Object.values(TaskRunStatus) as [TaskRunStatus, ...TaskRunStatus[]]);
|
||||
|
||||
const RunListInputOptionsSchema = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
//filters
|
||||
tasks: z.array(z.string()).optional(),
|
||||
versions: z.array(z.string()).optional(),
|
||||
statuses: z.array(RunStatus).optional(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
// "any" (default) = run has at least one of `tags`; "all" = run has every tag.
|
||||
tagsMatch: z.enum(["any", "all"]).optional(),
|
||||
scheduleId: z.string().optional(),
|
||||
period: z.string().optional(),
|
||||
from: z.number().optional(),
|
||||
to: z.number().optional(),
|
||||
isTest: z.boolean().optional(),
|
||||
rootOnly: z.boolean().optional(),
|
||||
batchId: z.string().optional(),
|
||||
runId: z.array(z.string()).optional(),
|
||||
bulkId: z.string().optional(),
|
||||
queues: z.array(z.string()).optional(),
|
||||
regions: z.array(z.string()).optional(),
|
||||
machines: MachinePresetName.array().optional(),
|
||||
errorId: z.string().optional(),
|
||||
taskKinds: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
export type RunListInputOptions = z.infer<typeof RunListInputOptionsSchema>;
|
||||
export type RunListInputFilters = Omit<
|
||||
RunListInputOptions,
|
||||
"organizationId" | "projectId" | "environmentId"
|
||||
>;
|
||||
|
||||
export type ParsedRunFilters = RunListInputFilters & {
|
||||
cursor?: string;
|
||||
direction?: "forward" | "backward";
|
||||
sources?: string[];
|
||||
};
|
||||
|
||||
export type FilterRunsOptions = Omit<RunListInputOptions, "period"> & {
|
||||
period: number | undefined;
|
||||
};
|
||||
|
||||
type Pagination = {
|
||||
page: {
|
||||
size: number;
|
||||
cursor?: string;
|
||||
direction?: "forward" | "backward";
|
||||
};
|
||||
};
|
||||
|
||||
type OffsetPagination = {
|
||||
offset: number;
|
||||
limit: number;
|
||||
};
|
||||
|
||||
export type ListedRun = Prisma.TaskRunGetPayload<{
|
||||
select: {
|
||||
id: true;
|
||||
friendlyId: true;
|
||||
taskIdentifier: true;
|
||||
taskVersion: true;
|
||||
runtimeEnvironmentId: true;
|
||||
status: true;
|
||||
createdAt: true;
|
||||
startedAt: true;
|
||||
lockedAt: true;
|
||||
delayUntil: true;
|
||||
updatedAt: true;
|
||||
completedAt: true;
|
||||
isTest: true;
|
||||
spanId: true;
|
||||
idempotencyKey: true;
|
||||
ttl: true;
|
||||
expiredAt: true;
|
||||
costInCents: true;
|
||||
baseCostInCents: true;
|
||||
usageDurationMs: true;
|
||||
runTags: true;
|
||||
depth: true;
|
||||
rootTaskRunId: true;
|
||||
batchId: true;
|
||||
metadata: true;
|
||||
metadataType: true;
|
||||
machinePreset: true;
|
||||
queue: true;
|
||||
workerQueue: true;
|
||||
region: true;
|
||||
annotations: true;
|
||||
};
|
||||
}>;
|
||||
|
||||
export type ListRunsOptions = RunListInputOptions & Pagination;
|
||||
|
||||
export type TagListOptions = {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
period?: string;
|
||||
from?: number;
|
||||
to?: number;
|
||||
/** Performs a case insensitive contains search on the tag name */
|
||||
query?: string;
|
||||
} & OffsetPagination;
|
||||
|
||||
export type TagList = {
|
||||
tags: string[];
|
||||
};
|
||||
|
||||
export type CursorPagination = {
|
||||
nextCursor: string | null;
|
||||
previousCursor: string | null;
|
||||
};
|
||||
|
||||
export type RunIdsPage = {
|
||||
runIds: string[];
|
||||
pagination: CursorPagination;
|
||||
};
|
||||
|
||||
export interface IRunsRepository {
|
||||
name: string;
|
||||
/**
|
||||
* A keyset-paginated page of run ids plus the cursors to navigate
|
||||
* forward/backward. The cursors are opaque composite `(created_at, run_id)`
|
||||
* tokens, so pagination can't duplicate or skip runs. This is the single
|
||||
* cursor-aware list primitive — `listRuns` and bulk actions build on it.
|
||||
*/
|
||||
listRunIds(options: ListRunsOptions): Promise<RunIdsPage>;
|
||||
/** Returns friendly IDs (e.g., run_xxx) instead of internal UUIDs. Used for ClickHouse task_events queries. */
|
||||
listFriendlyRunIds(options: ListRunsOptions): Promise<string[]>;
|
||||
listRuns(options: ListRunsOptions): Promise<{
|
||||
runs: ListedRun[];
|
||||
pagination: {
|
||||
nextCursor: string | null;
|
||||
previousCursor: string | null;
|
||||
};
|
||||
}>;
|
||||
countRuns(options: RunListInputOptions): Promise<number>;
|
||||
listTags(options: TagListOptions): Promise<TagList>;
|
||||
runExistsInEnvironment(options: {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
createdAtLowerBoundMs?: number;
|
||||
}): Promise<boolean>;
|
||||
}
|
||||
|
||||
export class RunsRepository implements IRunsRepository {
|
||||
private readonly clickHouseRunsRepository: ClickHouseRunsRepository;
|
||||
|
||||
constructor(private readonly options: RunsRepositoryOptions) {
|
||||
this.clickHouseRunsRepository = new ClickHouseRunsRepository(options);
|
||||
}
|
||||
|
||||
get name() {
|
||||
return "runsRepository";
|
||||
}
|
||||
|
||||
async runExistsInEnvironment(options: {
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
createdAtLowerBoundMs?: number;
|
||||
}): Promise<boolean> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.runExistsInEnvironment",
|
||||
async () => this.clickHouseRunsRepository.runExistsInEnvironment(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async listRunIds(options: ListRunsOptions): Promise<RunIdsPage> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.listRunIds",
|
||||
async () => this.clickHouseRunsRepository.listRunIds(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async listFriendlyRunIds(options: ListRunsOptions): Promise<string[]> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.listFriendlyRunIds",
|
||||
async () => this.clickHouseRunsRepository.listFriendlyRunIds(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
"readThrough.split": Boolean(this.options.readThrough?.splitEnabled),
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async listRuns(options: ListRunsOptions): Promise<{
|
||||
runs: ListedRun[];
|
||||
pagination: {
|
||||
nextCursor: string | null;
|
||||
previousCursor: string | null;
|
||||
};
|
||||
}> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.listRuns",
|
||||
async () => this.clickHouseRunsRepository.listRuns(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
"readThrough.split": Boolean(this.options.readThrough?.splitEnabled),
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async countRuns(options: RunListInputOptions): Promise<number> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.countRuns",
|
||||
async () => this.clickHouseRunsRepository.countRuns(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async listTags(options: TagListOptions): Promise<TagList> {
|
||||
return startActiveSpan(
|
||||
"runsRepository.listTags",
|
||||
async () => this.clickHouseRunsRepository.listTags(options),
|
||||
{
|
||||
attributes: {
|
||||
"repository.name": "clickhouse",
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
environmentId: options.environmentId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseRunListInputOptions(data: any): RunListInputOptions {
|
||||
return RunListInputOptionsSchema.parse(data);
|
||||
}
|
||||
|
||||
export async function convertRunListInputOptionsToFilterRunsOptions(
|
||||
options: RunListInputOptions,
|
||||
prisma: RunsRepositoryOptions["prisma"],
|
||||
store: RunStore = defaultRunStore
|
||||
): Promise<FilterRunsOptions> {
|
||||
const convertedOptions: FilterRunsOptions = {
|
||||
...options,
|
||||
period: undefined,
|
||||
};
|
||||
|
||||
// Convert time period to ms
|
||||
const time = timeFilters({
|
||||
period: options.period,
|
||||
from: options.from,
|
||||
to: options.to,
|
||||
});
|
||||
convertedOptions.period = time.period ? (parseDuration(time.period) ?? undefined) : undefined;
|
||||
|
||||
// Cross-DB resolution: BatchTaskRun is a RUN-OPS table. A run-ops batch resident on the
|
||||
// dedicated run-ops DB must resolve via the store's NEW->LEGACY probe — a single control-plane
|
||||
// client would miss it and leave the friendlyId in the ClickHouse `batch_id` filter, matching
|
||||
// nothing. Split off / self-host: the store is a passthrough over the one client.
|
||||
if (options.batchId && options.batchId.startsWith("batch_")) {
|
||||
const batch = await store.findBatchTaskRunByFriendlyId(options.batchId, options.environmentId);
|
||||
|
||||
if (batch) {
|
||||
convertedOptions.batchId = batch.id;
|
||||
}
|
||||
}
|
||||
|
||||
// ScheduleId can be a friendlyId. TaskSchedule is a CONTROL-PLANE table, so this stays on
|
||||
// the passed `prisma` (the control-plane client) in both single-DB and split modes.
|
||||
if (options.scheduleId && options.scheduleId.startsWith("sched_")) {
|
||||
const schedule = await prisma.taskSchedule.findFirst({
|
||||
select: {
|
||||
id: true,
|
||||
},
|
||||
where: {
|
||||
friendlyId: options.scheduleId,
|
||||
projectId: options.projectId,
|
||||
},
|
||||
});
|
||||
|
||||
if (schedule) {
|
||||
convertedOptions.scheduleId = schedule?.id;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.bulkId && options.bulkId.startsWith("bulk_")) {
|
||||
convertedOptions.bulkId = BulkActionId.toId(options.bulkId);
|
||||
}
|
||||
|
||||
if (options.runId) {
|
||||
// Convert to friendlyId
|
||||
convertedOptions.runId = options.runId.map((r) => RunId.toFriendlyId(r));
|
||||
}
|
||||
|
||||
// batchId/runId/scheduleId target specific runs, so rootOnly is meaningless and forced off.
|
||||
// tasks is intentionally excluded so rootOnly can narrow a task filter to root runs only.
|
||||
if (options.batchId || options.runId?.length || options.scheduleId) {
|
||||
convertedOptions.rootOnly = false;
|
||||
}
|
||||
|
||||
return convertedOptions;
|
||||
}
|
||||
Reference in New Issue
Block a user