wehub-resource-sync
98 lines
3.7 KiB
TypeScript
98 lines
3.7 KiB
TypeScript
import {
|
|
type PendingVersionRunIdLookup,
|
|
type PendingVersionRunIdLookupOptions,
|
|
type PendingVersionRunIdLookupResult,
|
|
} from "@internal/run-engine";
|
|
import type { Logger } from "@trigger.dev/core/logger";
|
|
import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
|
|
|
|
export type ClickhousePendingVersionLookupOptions = {
|
|
clickhouseFactory: ClickhouseFactory;
|
|
logger: Logger;
|
|
};
|
|
|
|
/**
|
|
* ClickHouse-backed lookup for `PENDING_VERSION` TaskRun ids.
|
|
*
|
|
* Resolves the ClickHouse client per call via the
|
|
* {@link ClickhouseFactory}, which honors per-organization data-store
|
|
* routing (HIPAA / data-sovereignty customers get their own instance,
|
|
* everyone else lands on the shared `engine` client configured by
|
|
* `RUN_ENGINE_CLICKHOUSE_*` env vars).
|
|
*
|
|
* Best-effort by design: replication lag against `task_runs_v2` can
|
|
* produce stale candidates. The run-engine consumer re-validates every
|
|
* id against Postgres by primary key with a `status = 'PENDING_VERSION'`
|
|
* guard before any mutation, so stale ids are dropped at the source of
|
|
* truth. On ClickHouse error we log and return an empty result; the
|
|
* pending-version re-enqueue tail loop retries on the next event.
|
|
*/
|
|
export class ClickhousePendingVersionLookup implements PendingVersionRunIdLookup {
|
|
readonly name = "clickhouse";
|
|
|
|
constructor(private readonly opts: ClickhousePendingVersionLookupOptions) {}
|
|
|
|
async lookupPendingVersionRunIds(
|
|
options: PendingVersionRunIdLookupOptions
|
|
): Promise<PendingVersionRunIdLookupResult> {
|
|
// Empty IN-lists would be a no-op; bail before issuing the query.
|
|
if (options.taskIdentifiers.length === 0 || options.queues.length === 0) {
|
|
return { runIds: [] };
|
|
}
|
|
|
|
let clickhouse;
|
|
try {
|
|
clickhouse = await this.opts.clickhouseFactory.getClickhouseForOrganization(
|
|
options.organizationId,
|
|
"engine"
|
|
);
|
|
} catch (error) {
|
|
// Factory resolution failures usually mean a real configuration
|
|
// problem (registry misload, missing data store, ClientType mismatch).
|
|
// These are not transient — log at error so ops sees them in dashboards
|
|
// and incident hooks. Query-level errors below stay at warn because
|
|
// those are expected to be transient.
|
|
this.opts.logger.error("ClickhousePendingVersionLookup factory resolution failed", {
|
|
error,
|
|
organizationId: options.organizationId,
|
|
});
|
|
return { runIds: [] };
|
|
}
|
|
|
|
const builder = clickhouse.taskRuns
|
|
.pendingVersionIdsQueryBuilder()
|
|
// `organization_id` MUST be the leading filter — it is the leading
|
|
// sort-key column on `task_runs_v2` and the only thing that prunes
|
|
// granules cheaply on a multi-tenant table.
|
|
.where("organization_id = {organizationId: String}", {
|
|
organizationId: options.organizationId,
|
|
})
|
|
.where("project_id = {projectId: String}", { projectId: options.projectId })
|
|
.where("environment_id = {environmentId: String}", {
|
|
environmentId: options.environmentId,
|
|
})
|
|
.where("status = 'PENDING_VERSION'")
|
|
.where("task_identifier IN {taskIdentifiers: Array(String)}", {
|
|
taskIdentifiers: options.taskIdentifiers,
|
|
})
|
|
.where("queue IN {queues: Array(String)}", { queues: options.queues })
|
|
.where("_is_deleted = 0")
|
|
.orderBy("created_at ASC")
|
|
.limit(options.limit);
|
|
|
|
const [queryError, rows] = await builder.execute();
|
|
|
|
if (queryError) {
|
|
this.opts.logger.warn("ClickhousePendingVersionLookup query failed", {
|
|
error: queryError,
|
|
organizationId: options.organizationId,
|
|
projectId: options.projectId,
|
|
environmentId: options.environmentId,
|
|
});
|
|
return { runIds: [] };
|
|
}
|
|
|
|
return { runIds: rows.map((row) => row.run_id) };
|
|
}
|
|
}
|