chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,251 @@
|
||||
import { RunEngine } from "@internal/run-engine";
|
||||
import { $replica, prisma } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { createBatchGlobalRateLimiter } from "~/runEngine/concerns/batchGlobalRateLimiter.server";
|
||||
import { SCHEDULED_WORKER_QUEUE_SUFFIX } from "~/runEngine/concerns/workerQueueSplit.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { defaultMachine, getCurrentPlan } from "~/services/platform.v3.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { allMachines } from "./machinePresets.server";
|
||||
import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.server";
|
||||
import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server";
|
||||
import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server";
|
||||
import { runStore } from "./runStore.server";
|
||||
import { meter, tracer } from "./tracer.server";
|
||||
|
||||
export const engine = singleton("RunEngine", createRunEngine);
|
||||
|
||||
export type { RunEngine };
|
||||
|
||||
function createRunEngine() {
|
||||
const engine = new RunEngine({
|
||||
prisma,
|
||||
readOnlyPrisma: $replica,
|
||||
crossSeamGuard: pickRunOpsStoreForCompletion,
|
||||
// Inject the shared run-store singleton so the engine and the webapp presenters/
|
||||
// services route through ONE store. When split is off this is the same passthrough
|
||||
// PostgresRunStore the engine would have defaulted to, so behavior is unchanged.
|
||||
store: runStore,
|
||||
controlPlaneResolver: runEngineControlPlaneResolver,
|
||||
logLevel: env.RUN_ENGINE_WORKER_LOG_LEVEL,
|
||||
treatProductionExecutionStallsAsOOM:
|
||||
env.RUN_ENGINE_TREAT_PRODUCTION_EXECUTION_STALLS_AS_OOM === "1",
|
||||
readReplicaSnapshotsSinceEnabled: env.RUN_ENGINE_READ_REPLICA_SNAPSHOTS_SINCE_ENABLED === "1",
|
||||
readReplicaSnapshotsSinceRetryDelay: {
|
||||
minMs: env.RUN_ENGINE_SNAPSHOTS_SINCE_REPLICA_RETRY_MIN_MS,
|
||||
maxMs: env.RUN_ENGINE_SNAPSHOTS_SINCE_REPLICA_RETRY_MAX_MS,
|
||||
},
|
||||
worker: {
|
||||
disabled: env.RUN_ENGINE_WORKER_ENABLED === "0",
|
||||
workers: env.RUN_ENGINE_WORKER_COUNT,
|
||||
tasksPerWorker: env.RUN_ENGINE_TASKS_PER_WORKER,
|
||||
pollIntervalMs: env.RUN_ENGINE_WORKER_POLL_INTERVAL,
|
||||
immediatePollIntervalMs: env.RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL,
|
||||
limit: env.RUN_ENGINE_WORKER_CONCURRENCY_LIMIT,
|
||||
shutdownTimeoutMs: env.RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS,
|
||||
redis: {
|
||||
keyPrefix: "engine:",
|
||||
port: env.RUN_ENGINE_WORKER_REDIS_PORT ?? undefined,
|
||||
host: env.RUN_ENGINE_WORKER_REDIS_HOST ?? undefined,
|
||||
username: env.RUN_ENGINE_WORKER_REDIS_USERNAME ?? undefined,
|
||||
password: env.RUN_ENGINE_WORKER_REDIS_PASSWORD ?? undefined,
|
||||
enableAutoPipelining: true,
|
||||
...(env.RUN_ENGINE_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
},
|
||||
machines: {
|
||||
defaultMachine,
|
||||
machines: allMachines(),
|
||||
baseCostInCents: env.CENTS_PER_RUN,
|
||||
},
|
||||
queue: {
|
||||
defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT,
|
||||
defaultEnvConcurrencyBurstFactor: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_BURST_FACTOR,
|
||||
logLevel: env.RUN_ENGINE_RUN_QUEUE_LOG_LEVEL,
|
||||
redis: {
|
||||
keyPrefix: "engine:",
|
||||
port: env.RUN_ENGINE_RUN_QUEUE_REDIS_PORT ?? undefined,
|
||||
host: env.RUN_ENGINE_RUN_QUEUE_REDIS_HOST ?? undefined,
|
||||
username: env.RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME ?? undefined,
|
||||
password: env.RUN_ENGINE_RUN_QUEUE_REDIS_PASSWORD ?? undefined,
|
||||
enableAutoPipelining: true,
|
||||
...(env.RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
queueSelectionStrategyOptions: {
|
||||
parentQueueLimit: env.RUN_ENGINE_PARENT_QUEUE_LIMIT,
|
||||
biases: {
|
||||
concurrencyLimitBias: env.RUN_ENGINE_CONCURRENCY_LIMIT_BIAS,
|
||||
availableCapacityBias: env.RUN_ENGINE_AVAILABLE_CAPACITY_BIAS,
|
||||
queueAgeRandomization: env.RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS,
|
||||
},
|
||||
reuseSnapshotCount: env.RUN_ENGINE_REUSE_SNAPSHOT_COUNT,
|
||||
maximumEnvCount: env.RUN_ENGINE_MAXIMUM_ENV_COUNT,
|
||||
tracer,
|
||||
},
|
||||
shardCount: env.RUN_ENGINE_RUN_QUEUE_SHARD_COUNT,
|
||||
processWorkerQueueDebounceMs: env.RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS,
|
||||
dequeueBlockingTimeoutSeconds: env.RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS,
|
||||
masterQueueConsumersIntervalMs: env.RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS,
|
||||
masterQueueConsumersDisabled: env.RUN_ENGINE_WORKER_ENABLED === "0",
|
||||
masterQueueCooloffPeriodMs: env.RUN_ENGINE_MASTER_QUEUE_COOLOFF_PERIOD_MS,
|
||||
masterQueueCooloffCountThreshold: env.RUN_ENGINE_MASTER_QUEUE_COOLOFF_COUNT_THRESHOLD,
|
||||
masterQueueConsumerDequeueCount: env.RUN_ENGINE_MASTER_QUEUE_CONSUMER_DEQUEUE_COUNT,
|
||||
concurrencySweeper: {
|
||||
scanSchedule: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_SCHEDULE,
|
||||
processMarkedSchedule: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE,
|
||||
scanJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS,
|
||||
processMarkedJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS,
|
||||
},
|
||||
ttlSystem: {
|
||||
disabled: env.RUN_ENGINE_TTL_SYSTEM_DISABLED,
|
||||
consumersDisabled: env.RUN_ENGINE_TTL_CONSUMERS_DISABLED,
|
||||
shardCount: env.RUN_ENGINE_TTL_SYSTEM_SHARD_COUNT,
|
||||
pollIntervalMs: env.RUN_ENGINE_TTL_SYSTEM_POLL_INTERVAL_MS,
|
||||
batchSize: env.RUN_ENGINE_TTL_SYSTEM_BATCH_SIZE,
|
||||
workerConcurrency: env.RUN_ENGINE_TTL_WORKER_CONCURRENCY,
|
||||
batchMaxSize: env.RUN_ENGINE_TTL_WORKER_BATCH_MAX_SIZE,
|
||||
batchMaxWaitMs: env.RUN_ENGINE_TTL_WORKER_BATCH_MAX_WAIT_MS,
|
||||
},
|
||||
},
|
||||
runLock: {
|
||||
redis: {
|
||||
keyPrefix: "engine:",
|
||||
port: env.RUN_ENGINE_RUN_LOCK_REDIS_PORT ?? undefined,
|
||||
host: env.RUN_ENGINE_RUN_LOCK_REDIS_HOST ?? undefined,
|
||||
username: env.RUN_ENGINE_RUN_LOCK_REDIS_USERNAME ?? undefined,
|
||||
password: env.RUN_ENGINE_RUN_LOCK_REDIS_PASSWORD ?? undefined,
|
||||
enableAutoPipelining: true,
|
||||
...(env.RUN_ENGINE_RUN_LOCK_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
duration: env.RUN_ENGINE_RUN_LOCK_DURATION,
|
||||
automaticExtensionThreshold: env.RUN_ENGINE_RUN_LOCK_AUTOMATIC_EXTENSION_THRESHOLD,
|
||||
retryConfig: {
|
||||
maxAttempts: env.RUN_ENGINE_RUN_LOCK_MAX_RETRIES,
|
||||
baseDelay: env.RUN_ENGINE_RUN_LOCK_BASE_DELAY,
|
||||
maxDelay: env.RUN_ENGINE_RUN_LOCK_MAX_DELAY,
|
||||
backoffMultiplier: env.RUN_ENGINE_RUN_LOCK_BACKOFF_MULTIPLIER,
|
||||
jitterFactor: env.RUN_ENGINE_RUN_LOCK_JITTER_FACTOR,
|
||||
maxTotalWaitTime: env.RUN_ENGINE_RUN_LOCK_MAX_TOTAL_WAIT_TIME,
|
||||
},
|
||||
},
|
||||
tracer,
|
||||
meter,
|
||||
workerQueueObserver: {
|
||||
enabled: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_ENABLED === "1",
|
||||
intervalMs: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_INTERVAL_MS,
|
||||
// Also observe the scheduled split variant of each worker queue. The suffix
|
||||
// naming convention lives in the webapp, so it is passed in here.
|
||||
additionalQueueSuffixes: [SCHEDULED_WORKER_QUEUE_SUFFIX],
|
||||
excludedCloudProviders: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_EXCLUDED_CLOUD_PROVIDERS.split(
|
||||
","
|
||||
)
|
||||
.map((provider) => provider.trim())
|
||||
.filter(Boolean),
|
||||
},
|
||||
defaultMaxTtl: env.RUN_ENGINE_DEFAULT_MAX_TTL,
|
||||
heartbeatTimeoutsMs: {
|
||||
PENDING_EXECUTING: env.RUN_ENGINE_TIMEOUT_PENDING_EXECUTING,
|
||||
PENDING_CANCEL: env.RUN_ENGINE_TIMEOUT_PENDING_CANCEL,
|
||||
EXECUTING: env.RUN_ENGINE_TIMEOUT_EXECUTING,
|
||||
EXECUTING_WITH_WAITPOINTS: env.RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS,
|
||||
SUSPENDED: env.RUN_ENGINE_TIMEOUT_SUSPENDED,
|
||||
},
|
||||
suspendedHeartbeatRetriesConfig: {
|
||||
maxCount: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_COUNT,
|
||||
maxDelayMs: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_DELAY_MS,
|
||||
initialDelayMs: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_INITIAL_DELAY_MS,
|
||||
factor: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_FACTOR,
|
||||
},
|
||||
retryWarmStartThresholdMs: env.RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS,
|
||||
pendingVersionRunIdLookup: runEnginePendingVersionLookup,
|
||||
billing: {
|
||||
getCurrentPlan: async (orgId: string) => {
|
||||
const plan = await getCurrentPlan(orgId);
|
||||
|
||||
// This only happens when there's no billing service running or on errors
|
||||
if (!plan) {
|
||||
logger.warn("engine.getCurrentPlan: no plan", { orgId });
|
||||
return {
|
||||
isPaying: true,
|
||||
type: "paid", // default to paid
|
||||
};
|
||||
}
|
||||
|
||||
// This shouldn't happen
|
||||
if (!plan.v3Subscription) {
|
||||
logger.warn("engine.getCurrentPlan: no v3 subscription", { orgId });
|
||||
return {
|
||||
isPaying: false,
|
||||
type: "free",
|
||||
};
|
||||
}
|
||||
|
||||
// Neither should this
|
||||
if (!plan.v3Subscription.plan) {
|
||||
logger.warn("engine.getCurrentPlan: no v3 subscription plan", { orgId });
|
||||
return {
|
||||
isPaying: plan.v3Subscription.isPaying,
|
||||
type: plan.v3Subscription.isPaying ? "paid" : "free",
|
||||
};
|
||||
}
|
||||
|
||||
// This is the normal case when the billing service is running
|
||||
return {
|
||||
isPaying: plan.v3Subscription.isPaying,
|
||||
type: plan.v3Subscription.plan.type,
|
||||
hasPrivateLink: plan.v3Subscription.plan.limits.hasPrivateNetworking ?? false,
|
||||
};
|
||||
},
|
||||
},
|
||||
// BatchQueue with DRR scheduling for fair batch processing
|
||||
// Consumers are controlled by options.worker.disabled (same as main worker)
|
||||
batchQueue: {
|
||||
redis: {
|
||||
keyPrefix: "engine:",
|
||||
port: env.BATCH_TRIGGER_WORKER_REDIS_PORT ?? undefined,
|
||||
host: env.BATCH_TRIGGER_WORKER_REDIS_HOST ?? undefined,
|
||||
username: env.BATCH_TRIGGER_WORKER_REDIS_USERNAME ?? undefined,
|
||||
password: env.BATCH_TRIGGER_WORKER_REDIS_PASSWORD ?? undefined,
|
||||
enableAutoPipelining: true,
|
||||
...(env.BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
drr: {
|
||||
quantum: env.BATCH_QUEUE_DRR_QUANTUM,
|
||||
maxDeficit: env.BATCH_QUEUE_MAX_DEFICIT,
|
||||
masterQueueLimit: env.BATCH_QUEUE_MASTER_QUEUE_LIMIT,
|
||||
},
|
||||
shardCount: env.BATCH_QUEUE_SHARD_COUNT,
|
||||
workerQueueBlockingTimeoutSeconds: env.BATCH_QUEUE_WORKER_QUEUE_ENABLED
|
||||
? env.BATCH_QUEUE_WORKER_QUEUE_TIMEOUT_SECONDS
|
||||
: undefined,
|
||||
consumerCount: env.BATCH_QUEUE_CONSUMER_COUNT,
|
||||
consumerIntervalMs: env.BATCH_QUEUE_CONSUMER_INTERVAL_MS,
|
||||
consumerEnabled: env.BATCH_QUEUE_WORKER_ENABLED,
|
||||
// Default processing concurrency when no specific limit is set
|
||||
// This is overridden per-batch based on the plan type at batch creation
|
||||
defaultConcurrency: env.BATCH_CONCURRENCY_LIMIT_DEFAULT,
|
||||
// Optional global rate limiter - limits max items/sec processed across all consumers
|
||||
globalRateLimiter: env.BATCH_QUEUE_GLOBAL_RATE_LIMIT
|
||||
? createBatchGlobalRateLimiter(env.BATCH_QUEUE_GLOBAL_RATE_LIMIT)
|
||||
: undefined,
|
||||
// Worker queue depth cap - prevents unbounded growth protecting visibility timeouts
|
||||
workerQueueMaxDepth: env.BATCH_QUEUE_WORKER_QUEUE_MAX_DEPTH,
|
||||
retry: {
|
||||
maxAttempts: 6,
|
||||
minTimeoutInMs: 1_000,
|
||||
maxTimeoutInMs: 30_000,
|
||||
factor: 2,
|
||||
randomize: true,
|
||||
},
|
||||
},
|
||||
// Debounce configuration
|
||||
debounce: {
|
||||
maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS,
|
||||
quantizeNewDelayUntilMs: env.RUN_ENGINE_DEBOUNCE_QUANTIZE_NEW_DELAY_UNTIL_MS,
|
||||
fastPathSkipEnabled: env.RUN_ENGINE_DEBOUNCE_FAST_PATH_SKIP_ENABLED === "1",
|
||||
useReplicaForFastPathRead: env.RUN_ENGINE_DEBOUNCE_USE_REPLICA_FOR_FAST_PATH_READ === "1",
|
||||
},
|
||||
});
|
||||
|
||||
return engine;
|
||||
}
|
||||
Reference in New Issue
Block a user