chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,231 @@
|
||||
import { BulkActionId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import {
|
||||
type Prisma,
|
||||
BulkActionNotificationType,
|
||||
BulkActionStatus,
|
||||
BulkActionType,
|
||||
type PrismaClient,
|
||||
type TaskRunStatus,
|
||||
} from "@trigger.dev/database";
|
||||
import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
|
||||
import { prisma } from "~/db.server";
|
||||
import type { RunsRepository } from "~/services/runsRepository/runsRepository.server";
|
||||
import {
|
||||
countInProgressRunsForBillableEnvironment,
|
||||
countQueuedRunsForBillableEnvironment,
|
||||
createBillingLimitRunsRepository,
|
||||
getBillableEnvironmentsForBillingLimit,
|
||||
} from "./billingLimitQueuedRuns.server";
|
||||
import { BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS } from "./billingLimitConstants";
|
||||
|
||||
export const BILLING_LIMIT_RESOLVE_CANCEL_SOURCE = "billing_limit_resolve_new_only";
|
||||
export const BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE = "billing_limit_in_progress";
|
||||
|
||||
export class BillingLimitBulkCancelIncompleteError extends Error {
|
||||
constructor(readonly bulkActionId: string) {
|
||||
super(`Billing limit bulk cancel did not complete within time budget: ${bulkActionId}`);
|
||||
this.name = "BillingLimitBulkCancelIncompleteError";
|
||||
}
|
||||
}
|
||||
|
||||
type BulkCancelSource =
|
||||
| typeof BILLING_LIMIT_RESOLVE_CANCEL_SOURCE
|
||||
| typeof BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE;
|
||||
|
||||
export type BillingLimitBulkCancelDeps = {
|
||||
prismaClient?: PrismaClient;
|
||||
createRunsRepository?: (organizationId: string) => Promise<RunsRepository>;
|
||||
enqueueProcessBulkAction?: (bulkActionId: string) => Promise<unknown>;
|
||||
processBulkActionToCompletion?: (
|
||||
bulkActionId: string,
|
||||
options?: { deadline?: number }
|
||||
) => Promise<{ completed: boolean }>;
|
||||
};
|
||||
|
||||
function resolveBulkCancelDeps(deps?: BillingLimitBulkCancelDeps) {
|
||||
return {
|
||||
prismaClient: deps?.prismaClient ?? prisma,
|
||||
createRunsRepository: deps?.createRunsRepository ?? createBillingLimitRunsRepository,
|
||||
enqueueProcessBulkAction:
|
||||
deps?.enqueueProcessBulkAction ??
|
||||
(async (bulkActionId: string) => {
|
||||
// Imported dynamically for the same reason as BulkActionService below:
|
||||
// commonWorker.server transitively loads marqs -> the
|
||||
// TaskRunConcurrencyTracker singleton, which throws when REDIS_HOST/
|
||||
// REDIS_PORT are unset (e.g. the webapp unit-test CI job).
|
||||
const { commonWorker } = await import("~/v3/commonWorker.server");
|
||||
await commonWorker.enqueue({
|
||||
id: `processBulkAction-${bulkActionId}`,
|
||||
job: "processBulkAction",
|
||||
payload: { bulkActionId },
|
||||
});
|
||||
}),
|
||||
processBulkActionToCompletion:
|
||||
deps?.processBulkActionToCompletion ??
|
||||
(async (bulkActionId: string, options?: { deadline?: number }) => {
|
||||
// Imported dynamically so this module doesn't eagerly load BulkActionV2 ->
|
||||
// CancelTaskRunService -> marqs -> the TaskRunConcurrencyTracker singleton,
|
||||
// which throws when REDIS_HOST/REDIS_PORT are unset (e.g. the webapp
|
||||
// unit-test CI job).
|
||||
const { BulkActionService } = await import("~/v3/services/bulk/BulkActionV2.server");
|
||||
const service = new BulkActionService();
|
||||
return service.processToCompletion(bulkActionId, { deadline: options?.deadline });
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export class BillingLimitBulkCancelService {
|
||||
static async cancelQueuedRuns(
|
||||
organizationId: string,
|
||||
options?: {
|
||||
dedupeKey?: string;
|
||||
waitForCompletion?: boolean;
|
||||
bulkCancelDeadline?: number;
|
||||
},
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return this.cancelRunsForBillableEnvironments(
|
||||
organizationId,
|
||||
{
|
||||
source: BILLING_LIMIT_RESOLVE_CANCEL_SOURCE,
|
||||
statuses: [...QUEUED_STATUSES],
|
||||
name: "Billing limit resolve — cancel queued runs",
|
||||
countRuns: countQueuedRunsForBillableEnvironment,
|
||||
dedupeKey: options?.dedupeKey,
|
||||
waitForCompletion: options?.waitForCompletion,
|
||||
bulkCancelDeadline: options?.bulkCancelDeadline,
|
||||
},
|
||||
deps
|
||||
);
|
||||
}
|
||||
|
||||
static async cancelInProgressRuns(
|
||||
organizationId: string,
|
||||
options: { hitAt: string },
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return this.cancelRunsForBillableEnvironments(
|
||||
organizationId,
|
||||
{
|
||||
source: BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE,
|
||||
statuses: [...RUNNING_STATUSES],
|
||||
name: "Billing limit hit — cancel in-progress runs",
|
||||
countRuns: countInProgressRunsForBillableEnvironment,
|
||||
dedupeKey: options.hitAt,
|
||||
},
|
||||
deps
|
||||
);
|
||||
}
|
||||
|
||||
private static async cancelRunsForBillableEnvironments(
|
||||
organizationId: string,
|
||||
options: {
|
||||
source: BulkCancelSource;
|
||||
statuses: TaskRunStatus[];
|
||||
name: string;
|
||||
countRuns: typeof countQueuedRunsForBillableEnvironment;
|
||||
dedupeKey?: string;
|
||||
waitForCompletion?: boolean;
|
||||
bulkCancelDeadline?: number;
|
||||
},
|
||||
deps?: BillingLimitBulkCancelDeps
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
const {
|
||||
prismaClient,
|
||||
createRunsRepository,
|
||||
enqueueProcessBulkAction,
|
||||
processBulkActionToCompletion,
|
||||
} = resolveBulkCancelDeps(deps);
|
||||
|
||||
const environments = await getBillableEnvironmentsForBillingLimit(organizationId, prismaClient);
|
||||
|
||||
if (environments.length === 0) {
|
||||
return { bulkActionIds: [] };
|
||||
}
|
||||
|
||||
const runsRepository = await createRunsRepository(organizationId);
|
||||
const bulkActionIds: string[] = [];
|
||||
const bulkActionInternalIds: string[] = [];
|
||||
|
||||
for (const environment of environments) {
|
||||
if (options.dedupeKey) {
|
||||
const existing = await prismaClient.bulkActionGroup.findFirst({
|
||||
where: {
|
||||
environmentId: environment.id,
|
||||
type: BulkActionType.CANCEL,
|
||||
dedupeKey: options.dedupeKey,
|
||||
status: { not: BulkActionStatus.ABORTED },
|
||||
},
|
||||
select: { id: true, friendlyId: true, status: true },
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
bulkActionIds.push(existing.friendlyId);
|
||||
|
||||
if (existing.status === BulkActionStatus.COMPLETED) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
bulkActionInternalIds.push(existing.id);
|
||||
} else {
|
||||
await enqueueProcessBulkAction(existing.id);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const count = await options.countRuns(runsRepository, organizationId, environment);
|
||||
|
||||
if (count === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { id, friendlyId } = BulkActionId.generate();
|
||||
|
||||
await prismaClient.bulkActionGroup.create({
|
||||
data: {
|
||||
id,
|
||||
friendlyId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
name: options.name,
|
||||
type: BulkActionType.CANCEL,
|
||||
dedupeKey: options.dedupeKey,
|
||||
params: {
|
||||
statuses: options.statuses,
|
||||
finalizeRun: true,
|
||||
source: options.source,
|
||||
...(options.dedupeKey ? { dedupeKey: options.dedupeKey } : {}),
|
||||
} as Prisma.InputJsonValue,
|
||||
queryName: "bulk_action_v1",
|
||||
totalCount: count,
|
||||
completionNotification: BulkActionNotificationType.NONE,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
bulkActionInternalIds.push(id);
|
||||
} else {
|
||||
await enqueueProcessBulkAction(id);
|
||||
}
|
||||
|
||||
bulkActionIds.push(friendlyId);
|
||||
}
|
||||
|
||||
if (options.waitForCompletion) {
|
||||
const deadline =
|
||||
options.bulkCancelDeadline ?? Date.now() + BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS;
|
||||
|
||||
for (const bulkActionId of bulkActionInternalIds) {
|
||||
const result = await processBulkActionToCompletion(bulkActionId, { deadline });
|
||||
if (!result.completed) {
|
||||
throw new BillingLimitBulkCancelIncompleteError(bulkActionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { bulkActionIds };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server";
|
||||
|
||||
export async function runBillingLimitCancelInProgressRuns(
|
||||
organizationId: string,
|
||||
hitAt: string
|
||||
): Promise<{ bulkActionIds: string[] }> {
|
||||
return BillingLimitBulkCancelService.cancelInProgressRuns(organizationId, { hitAt });
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { RuntimeEnvironmentType } from "@trigger.dev/database";
|
||||
|
||||
export const BILLABLE_ENVIRONMENT_TYPES = [
|
||||
"PRODUCTION",
|
||||
"STAGING",
|
||||
"PREVIEW",
|
||||
] as const satisfies RuntimeEnvironmentType[];
|
||||
|
||||
export type BillableEnvironmentType = (typeof BILLABLE_ENVIRONMENT_TYPES)[number];
|
||||
|
||||
export const BILLING_LIMIT_CONVERGE_BATCH_SIZE = 50;
|
||||
|
||||
/** Max concurrent per-org billing limit lookups during reconciliation. */
|
||||
export const BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY = 10;
|
||||
|
||||
/** Inline bulk-cancel budget for billing limit resolve (worker visibility is 10 min). */
|
||||
export const BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS = 8 * 60_000;
|
||||
|
||||
export type BillingLimitConvergeTargetState = "grace" | "rejected" | "ok";
|
||||
|
||||
export function isBillableEnvironmentType(type: RuntimeEnvironmentType): boolean {
|
||||
return (BILLABLE_ENVIRONMENT_TYPES as readonly RuntimeEnvironmentType[]).includes(type);
|
||||
}
|
||||
|
||||
export function buildBillingLimitResolveDedupeKey(
|
||||
organizationId: string,
|
||||
resolvedAt: string
|
||||
): string {
|
||||
return `billing-limit-resolve:${organizationId}:${resolvedAt}`;
|
||||
}
|
||||
|
||||
export function buildBillingLimitResolveJobId(organizationId: string, resolvedAt: string): string {
|
||||
return `billingLimit.resolve:${organizationId}:${resolvedAt}`;
|
||||
}
|
||||
|
||||
export function buildBillingLimitInProgressCancelJobId(
|
||||
organizationId: string,
|
||||
hitAt: string
|
||||
): string {
|
||||
return `billingLimit.cancelInProgress:${organizationId}:${hitAt}`;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import {
|
||||
EnvironmentPauseSource,
|
||||
type Organization,
|
||||
type PrismaClient,
|
||||
type Project,
|
||||
type RuntimeEnvironment,
|
||||
} from "@trigger.dev/database";
|
||||
import { prisma } from "~/db.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import {
|
||||
BILLABLE_ENVIRONMENT_TYPES,
|
||||
BILLING_LIMIT_CONVERGE_BATCH_SIZE,
|
||||
type BillingLimitConvergeTargetState,
|
||||
} from "./billingLimitConstants";
|
||||
|
||||
export type ConvergeOrgResult = {
|
||||
paused: number;
|
||||
unpaused: number;
|
||||
};
|
||||
|
||||
type EnvironmentWithRelations = RuntimeEnvironment & {
|
||||
organization: Organization;
|
||||
project: Project;
|
||||
};
|
||||
|
||||
type UpdateEnvConcurrency = (
|
||||
environment: EnvironmentWithRelations,
|
||||
maximumConcurrencyLimit?: number
|
||||
) => Promise<void>;
|
||||
|
||||
export async function convergeBillingLimitEnvironmentsForOrg(
|
||||
organizationId: string,
|
||||
targetState: BillingLimitConvergeTargetState,
|
||||
options?: {
|
||||
batchSize?: number;
|
||||
prismaClient?: PrismaClient;
|
||||
updateConcurrency?: UpdateEnvConcurrency;
|
||||
}
|
||||
): Promise<ConvergeOrgResult> {
|
||||
const db = options?.prismaClient ?? prisma;
|
||||
const batchSize = options?.batchSize ?? BILLING_LIMIT_CONVERGE_BATCH_SIZE;
|
||||
// Imported dynamically so this module (reachable from upsertBranch.server.ts at
|
||||
// module load) doesn't eagerly load runQueue.server -> marqs -> triggerTaskV1 ->
|
||||
// the autoIncrementCounter singleton, which throws when REDIS_HOST/REDIS_PORT are
|
||||
// unset (e.g. the webapp unit-test CI job).
|
||||
const updateConcurrency =
|
||||
options?.updateConcurrency ??
|
||||
(async (environment, maximumConcurrencyLimit) => {
|
||||
const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server");
|
||||
return updateEnvConcurrencyLimits(environment, maximumConcurrencyLimit);
|
||||
});
|
||||
|
||||
if (targetState === "ok") {
|
||||
return unpauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency);
|
||||
}
|
||||
|
||||
return pauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency);
|
||||
}
|
||||
|
||||
async function pauseBillingLimitEnvironments(
|
||||
organizationId: string,
|
||||
db: PrismaClient,
|
||||
batchSize: number,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
): Promise<ConvergeOrgResult> {
|
||||
let paused = 0;
|
||||
let cursor: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const environments = await db.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
type: { in: [...BILLABLE_ENVIRONMENT_TYPES] },
|
||||
paused: false,
|
||||
},
|
||||
take: batchSize,
|
||||
...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
|
||||
orderBy: { id: "asc" },
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (environments.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const environment of environments) {
|
||||
await pauseEnvironmentForBillingLimit(environment, db, updateConcurrency);
|
||||
paused++;
|
||||
}
|
||||
|
||||
cursor = environments[environments.length - 1]?.id;
|
||||
if (environments.length < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Billing limit converge paused environments", {
|
||||
organizationId,
|
||||
paused,
|
||||
});
|
||||
|
||||
return { paused, unpaused: 0 };
|
||||
}
|
||||
|
||||
async function unpauseBillingLimitEnvironments(
|
||||
organizationId: string,
|
||||
db: PrismaClient,
|
||||
batchSize: number,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
): Promise<ConvergeOrgResult> {
|
||||
let unpaused = 0;
|
||||
let cursor: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const environments = await db.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
take: batchSize,
|
||||
...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
|
||||
orderBy: { id: "asc" },
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (environments.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const environment of environments) {
|
||||
await resumeEnvironmentFromBillingLimit(environment, db, updateConcurrency);
|
||||
unpaused++;
|
||||
}
|
||||
|
||||
cursor = environments[environments.length - 1]?.id;
|
||||
if (environments.length < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Billing limit converge unpaused environments", {
|
||||
organizationId,
|
||||
unpaused,
|
||||
});
|
||||
|
||||
return { paused: 0, unpaused };
|
||||
}
|
||||
|
||||
async function pauseEnvironmentForBillingLimit(
|
||||
environment: EnvironmentWithRelations,
|
||||
db: PrismaClient,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
) {
|
||||
const updated = await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await updateConcurrency(updated, 0);
|
||||
} catch (error) {
|
||||
await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: { paused: false, pauseSource: null },
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
// The env's paused state changed (or was rolled back); drop any cached copy either way.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeEnvironmentFromBillingLimit(
|
||||
environment: EnvironmentWithRelations,
|
||||
db: PrismaClient,
|
||||
updateConcurrency: UpdateEnvConcurrency
|
||||
) {
|
||||
const updated = await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: false,
|
||||
pauseSource: null,
|
||||
},
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await updateConcurrency(updated);
|
||||
} catch (error) {
|
||||
await db.runtimeEnvironment.update({
|
||||
where: { id: environment.id },
|
||||
data: {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
// The env's paused state changed (or was rolled back); drop any cached copy either way.
|
||||
controlPlaneResolver.invalidateEnvironment(environment.id);
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import { z } from "zod";
|
||||
import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server";
|
||||
import { runBillingLimitReconcileTick } from "./runBillingLimitReconcileTick.server";
|
||||
import { seedBillingLimitReconcileQueue } from "./billingLimitReconcileQueue.server";
|
||||
|
||||
const ConvergePayloadSchema = z.object({
|
||||
organizationId: z.string(),
|
||||
targetState: z.enum(["grace", "rejected", "ok"]),
|
||||
});
|
||||
|
||||
export class BillingLimitConvergeEnvironmentsService {
|
||||
static async seedReconcileQueue(organizationId: string) {
|
||||
await seedBillingLimitReconcileQueue(organizationId);
|
||||
}
|
||||
|
||||
static async runConverge(payload: z.infer<typeof ConvergePayloadSchema>) {
|
||||
const parsed = ConvergePayloadSchema.parse(payload);
|
||||
return convergeBillingLimitEnvironmentsForOrg(parsed.organizationId, parsed.targetState);
|
||||
}
|
||||
|
||||
static async runReconcileTick() {
|
||||
await runBillingLimitReconcileTick();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { bustBillingLimitCaches } from "~/services/platform.v3.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server";
|
||||
import { buildBillingLimitResolveDedupeKey } from "./billingLimitConstants";
|
||||
import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export async function convergeBillingLimitResolve(
|
||||
pending: PendingBillingLimitResolve
|
||||
): Promise<void> {
|
||||
const { organizationId, resumeMode, resolvedAt } = pending;
|
||||
|
||||
bustBillingLimitCaches(organizationId);
|
||||
|
||||
if (resumeMode === "new_only") {
|
||||
await BillingLimitBulkCancelService.cancelQueuedRuns(organizationId, {
|
||||
dedupeKey: buildBillingLimitResolveDedupeKey(organizationId, resolvedAt),
|
||||
waitForCompletion: true,
|
||||
});
|
||||
}
|
||||
|
||||
await convergeBillingLimitEnvironmentsForOrg(organizationId, "ok");
|
||||
|
||||
logger.info("Converged billing limit resolve", {
|
||||
organizationId,
|
||||
resumeMode,
|
||||
resolvedAt,
|
||||
});
|
||||
}
|
||||
|
||||
export { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server";
|
||||
@@ -0,0 +1,26 @@
|
||||
export type BillingLimitHitPayload = {
|
||||
organizationId: string;
|
||||
hitAt: string;
|
||||
cancelInProgressRuns: boolean;
|
||||
};
|
||||
|
||||
export type BillingLimitHitDeps = {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
seedReconcileQueue: (organizationId: string) => Promise<void>;
|
||||
enqueueConverge: (organizationId: string, targetState: "grace") => Promise<unknown>;
|
||||
enqueueCancelInProgressRuns: (organizationId: string, hitAt: string) => Promise<unknown>;
|
||||
};
|
||||
|
||||
/** Process billing limit grace hit from the billing platform webhook. */
|
||||
export async function processBillingLimitHit(
|
||||
payload: BillingLimitHitPayload,
|
||||
deps: BillingLimitHitDeps
|
||||
): Promise<void> {
|
||||
deps.bustCaches(payload.organizationId);
|
||||
await deps.seedReconcileQueue(payload.organizationId);
|
||||
await deps.enqueueConverge(payload.organizationId, "grace");
|
||||
|
||||
if (payload.cancelInProgressRuns) {
|
||||
await deps.enqueueCancelInProgressRuns(payload.organizationId, payload.hitAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type PendingBillingLimitResolve = {
|
||||
organizationId: string;
|
||||
resumeMode: "queue" | "new_only";
|
||||
resolvedAt: string;
|
||||
};
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { classifyPendingBillingLimitResolveConvergeFailure } from "./billingLimitPendingResolveFailure.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type RunPendingBillingLimitResolveDeps = {
|
||||
converge?: (pending: PendingBillingLimitResolve) => Promise<void>;
|
||||
complete?: (organizationId: string) => Promise<{ completed: boolean } | undefined>;
|
||||
};
|
||||
|
||||
export async function runPendingBillingLimitResolves(
|
||||
pendingResolves: PendingBillingLimitResolve[],
|
||||
deps: RunPendingBillingLimitResolveDeps = {}
|
||||
): Promise<Set<string>> {
|
||||
const converge =
|
||||
deps.converge ??
|
||||
(await import("./billingLimitConvergeResolve.server")).convergeBillingLimitResolve;
|
||||
const complete =
|
||||
deps.complete ?? (await import("~/services/platform.v3.server")).completeBillingLimitResolve;
|
||||
|
||||
const stillPendingOrgIds = new Set<string>();
|
||||
|
||||
for (const pending of pendingResolves) {
|
||||
try {
|
||||
await converge(pending);
|
||||
} catch (error) {
|
||||
logger.error("Failed to converge pending billing limit resolve", {
|
||||
failureClass: classifyPendingBillingLimitResolveConvergeFailure(pending.resumeMode),
|
||||
error,
|
||||
organizationId: pending.organizationId,
|
||||
resumeMode: pending.resumeMode,
|
||||
resolvedAt: pending.resolvedAt,
|
||||
});
|
||||
stillPendingOrgIds.add(pending.organizationId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const completion = await complete(pending.organizationId);
|
||||
if (!completion || completion.completed !== true) {
|
||||
throw new Error("Billing platform client unavailable");
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Failed to ack pending billing limit resolve", {
|
||||
failureClass: "ack-only",
|
||||
error,
|
||||
organizationId: pending.organizationId,
|
||||
resumeMode: pending.resumeMode,
|
||||
resolvedAt: pending.resolvedAt,
|
||||
});
|
||||
stillPendingOrgIds.add(pending.organizationId);
|
||||
}
|
||||
}
|
||||
|
||||
return stillPendingOrgIds;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
export type PendingBillingLimitResolveFailureClass =
|
||||
| "cancel-failing"
|
||||
| "converge-failing"
|
||||
| "ack-only";
|
||||
|
||||
/** Used in converge logs to classify stuck pending resolves. */
|
||||
export function classifyPendingBillingLimitResolveConvergeFailure(
|
||||
resumeMode: "queue" | "new_only"
|
||||
): Exclude<PendingBillingLimitResolveFailureClass, "ack-only"> {
|
||||
return resumeMode === "new_only" ? "cancel-failing" : "converge-failing";
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database";
|
||||
import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
|
||||
import { prisma } from "~/db.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
|
||||
import { BILLABLE_ENVIRONMENT_TYPES } from "./billingLimitConstants";
|
||||
|
||||
export type BillableEnvironmentRef = {
|
||||
id: string;
|
||||
projectId: string;
|
||||
};
|
||||
|
||||
export async function getBillableEnvironmentsForBillingLimit(
|
||||
organizationId: string,
|
||||
prismaClient: PrismaClient = prisma
|
||||
): Promise<BillableEnvironmentRef[]> {
|
||||
return prismaClient.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
organizationId,
|
||||
type: { in: [...BILLABLE_ENVIRONMENT_TYPES] },
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
projectId: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function createBillingLimitRunsRepository(organizationId: string) {
|
||||
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
|
||||
organizationId,
|
||||
"standard"
|
||||
);
|
||||
|
||||
return new RunsRepository({
|
||||
clickhouse,
|
||||
prisma: prisma as PrismaClient,
|
||||
});
|
||||
}
|
||||
|
||||
export async function countQueuedRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef
|
||||
): Promise<number> {
|
||||
return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [
|
||||
...QUEUED_STATUSES,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function countInProgressRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef
|
||||
): Promise<number> {
|
||||
return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [
|
||||
...RUNNING_STATUSES,
|
||||
]);
|
||||
}
|
||||
|
||||
async function countRunsForBillableEnvironment(
|
||||
runsRepository: RunsRepository,
|
||||
organizationId: string,
|
||||
environment: BillableEnvironmentRef,
|
||||
statuses: TaskRunStatus[]
|
||||
): Promise<number> {
|
||||
return runsRepository.countRuns({
|
||||
organizationId,
|
||||
projectId: environment.projectId,
|
||||
environmentId: environment.id,
|
||||
statuses,
|
||||
});
|
||||
}
|
||||
|
||||
/** Same source as BillingLimitBulkCancelService — ClickHouse countRuns(QUEUED_STATUSES). */
|
||||
export async function countBillableQueuedRunsForOrganization(
|
||||
organizationId: string
|
||||
): Promise<number> {
|
||||
const environments = await getBillableEnvironmentsForBillingLimit(organizationId);
|
||||
|
||||
if (environments.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const runsRepository = await createBillingLimitRunsRepository(organizationId);
|
||||
|
||||
let total = 0;
|
||||
|
||||
for (const environment of environments) {
|
||||
total += await countQueuedRunsForBillableEnvironment(
|
||||
runsRepository,
|
||||
organizationId,
|
||||
environment
|
||||
);
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { env } from "~/env.server";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
const RECONCILE_QUEUE_KEY = "billing-limit:reconcile-queue";
|
||||
|
||||
function createQueueRedis() {
|
||||
return createRedisClient("billing-limit:reconcile", {
|
||||
keyPrefix: "",
|
||||
host: env.BILLING_LIMIT_WORKER_REDIS_HOST,
|
||||
port: env.BILLING_LIMIT_WORKER_REDIS_PORT,
|
||||
username: env.BILLING_LIMIT_WORKER_REDIS_USERNAME,
|
||||
password: env.BILLING_LIMIT_WORKER_REDIS_PASSWORD,
|
||||
tlsDisabled: env.BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED === "true",
|
||||
});
|
||||
}
|
||||
|
||||
const queueRedis = singleton("billingLimitReconcileQueueRedis", createQueueRedis);
|
||||
|
||||
export async function seedBillingLimitReconcileQueue(organizationId: string): Promise<void> {
|
||||
await queueRedis.sadd(RECONCILE_QUEUE_KEY, organizationId);
|
||||
}
|
||||
|
||||
export async function readBillingLimitReconcileQueue(): Promise<string[]> {
|
||||
return queueRedis.smembers(RECONCILE_QUEUE_KEY);
|
||||
}
|
||||
|
||||
export async function removeFromBillingLimitReconcileQueue(
|
||||
organizationIds: string[]
|
||||
): Promise<void> {
|
||||
if (organizationIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
await queueRedis.srem(RECONCILE_QUEUE_KEY, ...organizationIds);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { BillingLimitConvergeTargetState } from "./billingLimitConstants";
|
||||
import type { OrgReconcileTarget } from "./billingLimitReconciliation.server";
|
||||
|
||||
export async function reconcileBillingLimitTarget(
|
||||
target: OrgReconcileTarget,
|
||||
deps: {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
enqueueConverge: (
|
||||
organizationId: string,
|
||||
targetState: BillingLimitConvergeTargetState
|
||||
) => Promise<unknown>;
|
||||
}
|
||||
) {
|
||||
// Safety net when webhooks are lost: bust stale entitlement after reject or resolve.
|
||||
if (target.targetState === "rejected" || target.targetState === "ok") {
|
||||
deps.bustCaches(target.organizationId);
|
||||
}
|
||||
|
||||
await deps.enqueueConverge(target.organizationId, target.targetState);
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import pMap from "p-map";
|
||||
import { prisma } from "~/db.server";
|
||||
import type { BillingLimitResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { getActiveBillingLimits, getBillingLimit } from "~/services/platform.v3.server";
|
||||
import {
|
||||
BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY,
|
||||
type BillingLimitConvergeTargetState,
|
||||
} from "./billingLimitConstants";
|
||||
import {
|
||||
readBillingLimitReconcileQueue,
|
||||
removeFromBillingLimitReconcileQueue,
|
||||
} from "./billingLimitReconcileQueue.server";
|
||||
|
||||
export type OrgReconcileTarget = {
|
||||
organizationId: string;
|
||||
targetState: BillingLimitConvergeTargetState;
|
||||
};
|
||||
|
||||
export function resolveConvergeTargetFromBillingLimit(
|
||||
billingLimit: BillingLimitResult | undefined
|
||||
): BillingLimitConvergeTargetState {
|
||||
if (!billingLimit?.isConfigured) {
|
||||
return "ok";
|
||||
}
|
||||
|
||||
if (billingLimit.limitState.status === "grace") {
|
||||
return "grace";
|
||||
}
|
||||
|
||||
if (billingLimit.limitState.status === "rejected") {
|
||||
return "rejected";
|
||||
}
|
||||
|
||||
return "ok";
|
||||
}
|
||||
|
||||
/** Reconcile path only — skip org when the platform lookup failed (undefined ≠ unconfigured). */
|
||||
export function resolveReconcileTargetFromBillingLimit(
|
||||
billingLimit: BillingLimitResult | undefined
|
||||
): BillingLimitConvergeTargetState | undefined {
|
||||
if (billingLimit === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return resolveConvergeTargetFromBillingLimit(billingLimit);
|
||||
}
|
||||
|
||||
export async function getOrgIdsWithBillingPauseSource(): Promise<string[]> {
|
||||
const rows = await prisma.runtimeEnvironment.findMany({
|
||||
where: {
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
select: {
|
||||
organizationId: true,
|
||||
},
|
||||
distinct: ["organizationId"],
|
||||
});
|
||||
|
||||
return rows.map((row) => row.organizationId);
|
||||
}
|
||||
|
||||
export function collectOrgIdsNeedingBillingLimitLookup(options: {
|
||||
staleOrgIds: string[];
|
||||
queuedOrgIds: string[];
|
||||
excludeOrgIds: Set<string>;
|
||||
coveredOrgIds: Set<string>;
|
||||
}): string[] {
|
||||
const orgIds = new Set<string>();
|
||||
|
||||
for (const organizationId of [...options.staleOrgIds, ...options.queuedOrgIds]) {
|
||||
if (options.excludeOrgIds.has(organizationId) || options.coveredOrgIds.has(organizationId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
orgIds.add(organizationId);
|
||||
}
|
||||
|
||||
return [...orgIds];
|
||||
}
|
||||
|
||||
export async function resolveReconcileTargetsForOrgLookups(
|
||||
organizationIds: string[],
|
||||
options?: {
|
||||
getBillingLimit?: (organizationId: string) => Promise<BillingLimitResult | undefined>;
|
||||
concurrency?: number;
|
||||
}
|
||||
): Promise<Map<string, BillingLimitConvergeTargetState>> {
|
||||
const lookupBillingLimit = options?.getBillingLimit ?? getBillingLimit;
|
||||
const concurrency = options?.concurrency ?? BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY;
|
||||
const targets = new Map<string, BillingLimitConvergeTargetState>();
|
||||
|
||||
await pMap(
|
||||
organizationIds,
|
||||
async (organizationId) => {
|
||||
try {
|
||||
const billingLimit = await lookupBillingLimit(organizationId);
|
||||
const targetState = resolveReconcileTargetFromBillingLimit(billingLimit);
|
||||
if (targetState === undefined) {
|
||||
logger.warn("Skipping billing limit reconcile — platform lookup unavailable", {
|
||||
organizationId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
targets.set(organizationId, targetState);
|
||||
} catch (error) {
|
||||
logger.error("Failed billing limit lookup for reconcile", {
|
||||
organizationId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
},
|
||||
{ concurrency }
|
||||
);
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
export async function collectOrgsToReconcile(options?: { excludeOrgIds?: Set<string> }): Promise<{
|
||||
targets: OrgReconcileTarget[];
|
||||
queuedOrgIds: string[];
|
||||
}> {
|
||||
const excludeOrgIds = options?.excludeOrgIds ?? new Set<string>();
|
||||
const targetByOrgId = new Map<string, BillingLimitConvergeTargetState>();
|
||||
|
||||
const activeLimits = await getActiveBillingLimits();
|
||||
if (activeLimits) {
|
||||
for (const org of activeLimits.orgs) {
|
||||
if (excludeOrgIds.has(org.orgId)) {
|
||||
continue;
|
||||
}
|
||||
targetByOrgId.set(org.orgId, org.limitState);
|
||||
}
|
||||
}
|
||||
|
||||
const [staleOrgIds, queuedOrgIds] = await Promise.all([
|
||||
getOrgIdsWithBillingPauseSource(),
|
||||
readBillingLimitReconcileQueue(),
|
||||
]);
|
||||
|
||||
const orgIdsNeedingLookup = collectOrgIdsNeedingBillingLimitLookup({
|
||||
staleOrgIds,
|
||||
queuedOrgIds,
|
||||
excludeOrgIds,
|
||||
coveredOrgIds: new Set(targetByOrgId.keys()),
|
||||
});
|
||||
|
||||
const lookedUpTargets = await resolveReconcileTargetsForOrgLookups(orgIdsNeedingLookup);
|
||||
for (const [organizationId, targetState] of lookedUpTargets) {
|
||||
targetByOrgId.set(organizationId, targetState);
|
||||
}
|
||||
|
||||
return {
|
||||
targets: Array.from(targetByOrgId.entries()).map(([organizationId, targetState]) => ({
|
||||
organizationId,
|
||||
targetState,
|
||||
})),
|
||||
queuedOrgIds,
|
||||
};
|
||||
}
|
||||
|
||||
export async function clearProcessedReconcileQueueEntries(
|
||||
queuedOrgIds: string[],
|
||||
processedOrgIds: string[]
|
||||
): Promise<void> {
|
||||
const processed = new Set(processedOrgIds);
|
||||
const toRemove = queuedOrgIds.filter((orgId) => processed.has(orgId));
|
||||
await removeFromBillingLimitReconcileQueue(toRemove);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
|
||||
export type BillingLimitResolveDeps = {
|
||||
bustCaches: (organizationId: string) => void;
|
||||
enqueueResolve: (pending: PendingBillingLimitResolve) => Promise<unknown>;
|
||||
};
|
||||
|
||||
/** Process billing limit resolve from the billing platform webhook. */
|
||||
export async function processBillingLimitResolve(
|
||||
pending: PendingBillingLimitResolve,
|
||||
deps: BillingLimitResolveDeps
|
||||
): Promise<void> {
|
||||
deps.bustCaches(pending.organizationId);
|
||||
await deps.enqueueResolve(pending);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import { prisma } from "~/db.server";
|
||||
import { countBillableQueuedRunsForOrganization } from "./billingLimitQueuedRuns.server";
|
||||
|
||||
export async function getBillingLimitQueuedRunCount(organizationId: string): Promise<number> {
|
||||
return countBillableQueuedRunsForOrganization(organizationId);
|
||||
}
|
||||
|
||||
export async function countBillingLimitPausedEnvironments(organizationId: string): Promise<number> {
|
||||
return prisma.runtimeEnvironment.count({
|
||||
where: {
|
||||
organizationId,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
},
|
||||
});
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import {
|
||||
EnvironmentPauseSource,
|
||||
type Organization,
|
||||
type Project,
|
||||
type RuntimeEnvironment,
|
||||
type RuntimeEnvironmentType,
|
||||
} from "@trigger.dev/database";
|
||||
import type { BillingLimitResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { isBillableEnvironmentType } from "./billingLimitConstants";
|
||||
import { resolveConvergeTargetFromBillingLimit } from "./billingLimitReconciliation.server";
|
||||
|
||||
export type InitialEnvPauseState = {
|
||||
paused: boolean;
|
||||
pauseSource: typeof EnvironmentPauseSource.BILLING_LIMIT | null;
|
||||
};
|
||||
|
||||
export type GetInitialEnvPauseStateDeps = {
|
||||
getBillingLimit?: (organizationId: string) => Promise<BillingLimitResult | undefined>;
|
||||
};
|
||||
|
||||
export async function getInitialEnvPauseStateForBillingLimit(
|
||||
organizationId: string,
|
||||
type: RuntimeEnvironmentType,
|
||||
deps: GetInitialEnvPauseStateDeps = {}
|
||||
): Promise<InitialEnvPauseState> {
|
||||
if (!isBillableEnvironmentType(type)) {
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
let billingLimit: BillingLimitResult | undefined;
|
||||
try {
|
||||
billingLimit = deps.getBillingLimit
|
||||
? await deps.getBillingLimit(organizationId)
|
||||
: await (await import("~/services/platform.v3.server")).getBillingLimit(organizationId);
|
||||
} catch (error) {
|
||||
logger.error("Failed to fetch billing limit for initial env pause state", {
|
||||
organizationId,
|
||||
error,
|
||||
});
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
const targetState = resolveConvergeTargetFromBillingLimit(billingLimit);
|
||||
|
||||
if (targetState === "grace" || targetState === "rejected") {
|
||||
return {
|
||||
paused: true,
|
||||
pauseSource: EnvironmentPauseSource.BILLING_LIMIT,
|
||||
};
|
||||
}
|
||||
|
||||
return { paused: false, pauseSource: null };
|
||||
}
|
||||
|
||||
export async function applyBillingLimitPauseAfterEnvCreate(
|
||||
environment: RuntimeEnvironment & { organization: Organization; project: Project }
|
||||
): Promise<void> {
|
||||
if (!environment.paused || environment.pauseSource !== EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Imported dynamically so this module (pulled in at module load by
|
||||
// upsertBranch.server.ts) doesn't eagerly load runQueue.server -> marqs ->
|
||||
// triggerTaskV1 -> the autoIncrementCounter singleton, which throws when
|
||||
// REDIS_HOST/REDIS_PORT are unset (e.g. the webapp unit-test CI job).
|
||||
const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server");
|
||||
await updateEnvConcurrencyLimits(environment, 0);
|
||||
} catch (error) {
|
||||
logger.error("Failed to apply billing-limit pause after env create", {
|
||||
environmentId: environment.id,
|
||||
organizationId: environment.organizationId,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { EnvironmentPauseSource } from "@trigger.dev/database";
|
||||
import type { PauseStatus } from "~/v3/services/pauseEnvironment.server";
|
||||
|
||||
/**
|
||||
* Guards manual pause/resume API calls while an environment is billing-paused.
|
||||
*
|
||||
* Design trade-off: billing-limit converge unpauses every environment with
|
||||
* `pauseSource=BILLING_LIMIT` on resolve. We therefore do not record a
|
||||
* separate manual pause on top of billing enforcement — a manual pause attempt
|
||||
* while already billing-paused is a silent no-op (`success: true`, still
|
||||
* paused). If the limit is later resolved, that environment is unpaused with
|
||||
* the rest, even if the caller intended to keep it paused.
|
||||
*
|
||||
* The queues UI hides pause/resume while `pauseSource=BILLING_LIMIT`; API
|
||||
* callers can still hit this path and should treat the no-op as idempotent.
|
||||
*/
|
||||
export function getManualPauseEnvironmentResult(
|
||||
action: PauseStatus,
|
||||
pauseSource: EnvironmentPauseSource | null | undefined
|
||||
):
|
||||
| { proceed: true }
|
||||
| { proceed: false; success: true; state: PauseStatus }
|
||||
| { proceed: false; success: false; error: string } {
|
||||
if (action === "resumed" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
return {
|
||||
proceed: false,
|
||||
success: false,
|
||||
error:
|
||||
"This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.",
|
||||
};
|
||||
}
|
||||
|
||||
if (action === "paused" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) {
|
||||
// Already billing-paused; do not overwrite pauseSource so resolve converge
|
||||
// can still find and unpause this environment.
|
||||
return {
|
||||
proceed: false,
|
||||
success: true,
|
||||
state: "paused",
|
||||
};
|
||||
}
|
||||
|
||||
return { proceed: true };
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { BillingLimitsPendingResolvesResult } from "~/services/billingLimit.schemas";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server";
|
||||
import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types";
|
||||
import type { OrgReconcileTarget } from "./billingLimitReconciliation.server";
|
||||
import type { reconcileBillingLimitTarget } from "./billingLimitReconcileTarget.server";
|
||||
|
||||
export type RunBillingLimitReconcileTickDeps = {
|
||||
getPendingResolves?: () => Promise<BillingLimitsPendingResolvesResult | undefined>;
|
||||
runPendingResolves?: (pendingResolves: PendingBillingLimitResolve[]) => Promise<Set<string>>;
|
||||
collectOrgs?: (options?: { excludeOrgIds?: Set<string> }) => Promise<{
|
||||
targets: OrgReconcileTarget[];
|
||||
queuedOrgIds: string[];
|
||||
}>;
|
||||
reconcileTarget?: typeof reconcileBillingLimitTarget;
|
||||
clearProcessedQueue?: (queuedOrgIds: string[], processedOrgIds: string[]) => Promise<void>;
|
||||
bustCaches?: (organizationId: string) => void;
|
||||
enqueueConverge?: (
|
||||
organizationId: string,
|
||||
targetState: OrgReconcileTarget["targetState"]
|
||||
) => Promise<void>;
|
||||
};
|
||||
|
||||
export async function runBillingLimitReconcileTick(
|
||||
deps: RunBillingLimitReconcileTickDeps = {}
|
||||
): Promise<void> {
|
||||
const getPendingResolves =
|
||||
deps.getPendingResolves ??
|
||||
(await import("~/services/platform.v3.server")).getPendingBillingLimitResolves;
|
||||
const runPendingResolves = deps.runPendingResolves ?? runPendingBillingLimitResolves;
|
||||
const collectOrgs =
|
||||
deps.collectOrgs ??
|
||||
(await import("./billingLimitReconciliation.server")).collectOrgsToReconcile;
|
||||
const reconcileTarget =
|
||||
deps.reconcileTarget ??
|
||||
(await import("./billingLimitReconcileTarget.server")).reconcileBillingLimitTarget;
|
||||
const clearProcessedQueue =
|
||||
deps.clearProcessedQueue ??
|
||||
(await import("./billingLimitReconciliation.server")).clearProcessedReconcileQueueEntries;
|
||||
const bustCaches =
|
||||
deps.bustCaches ?? (await import("~/services/platform.v3.server")).bustBillingLimitCaches;
|
||||
|
||||
const pendingResolves = (await getPendingResolves())?.orgs ?? [];
|
||||
const stillPendingOrgIds = await runPendingResolves(pendingResolves);
|
||||
|
||||
const { targets, queuedOrgIds } = await collectOrgs({
|
||||
excludeOrgIds: stillPendingOrgIds,
|
||||
});
|
||||
|
||||
const enqueueConverge =
|
||||
deps.enqueueConverge ??
|
||||
(async (organizationId, targetState) => {
|
||||
const { enqueueBillingLimitConverge } = await import("~/v3/billingLimitWorker.server");
|
||||
await enqueueBillingLimitConverge(organizationId, targetState);
|
||||
});
|
||||
|
||||
const processedOrgIds: string[] = [];
|
||||
for (const target of targets) {
|
||||
try {
|
||||
await reconcileTarget(target, {
|
||||
bustCaches,
|
||||
enqueueConverge,
|
||||
});
|
||||
processedOrgIds.push(target.organizationId);
|
||||
} catch (error) {
|
||||
logger.error("Failed to reconcile billing limit target", {
|
||||
organizationId: target.organizationId,
|
||||
targetState: target.targetState,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await clearProcessedQueue(queuedOrgIds, processedOrgIds);
|
||||
}
|
||||
Reference in New Issue
Block a user