chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,235 @@
|
||||
import invariant from "tiny-invariant";
|
||||
import { env } from "~/env.server";
|
||||
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
|
||||
import { meter, provider } from "~/v3/tracer.server";
|
||||
import {
|
||||
setRunsReplicationConfiguredSources,
|
||||
setRunsReplicationGlobal,
|
||||
} from "./runsReplicationGlobal.server";
|
||||
import {
|
||||
RunsReplicationService,
|
||||
type RunsReplicationSource,
|
||||
} from "./runsReplicationService.server";
|
||||
import { signalsEmitter } from "./signals.server";
|
||||
|
||||
export const runsReplicationInstance = singleton(
|
||||
"runsReplicationInstance",
|
||||
initializeRunsReplicationInstance
|
||||
);
|
||||
|
||||
export function buildReplicationSources(args: {
|
||||
splitEnabled: boolean;
|
||||
legacyUrl: string;
|
||||
newUrl?: string;
|
||||
/** `false` forces the new source off under split; undefined follows split. */
|
||||
newSourceOverride?: boolean;
|
||||
legacySlotName: string;
|
||||
legacyPublicationName: string;
|
||||
legacyOriginGeneration: number;
|
||||
newSlotName: string;
|
||||
newPublicationName: string;
|
||||
newOriginGeneration: number;
|
||||
}): RunsReplicationSource[] {
|
||||
const legacy: RunsReplicationSource = {
|
||||
id: "legacy",
|
||||
pgConnectionUrl: args.legacyUrl,
|
||||
slotName: args.legacySlotName,
|
||||
publicationName: args.legacyPublicationName,
|
||||
originGeneration: args.legacyOriginGeneration,
|
||||
};
|
||||
|
||||
const newSourceOn = args.splitEnabled && !!args.newUrl && args.newSourceOverride !== false;
|
||||
|
||||
if (!newSourceOn || !args.newUrl) {
|
||||
return [legacy];
|
||||
}
|
||||
|
||||
const next: RunsReplicationSource = {
|
||||
id: "new",
|
||||
pgConnectionUrl: args.newUrl,
|
||||
slotName: args.newSlotName,
|
||||
publicationName: args.newPublicationName,
|
||||
originGeneration: args.newOriginGeneration,
|
||||
};
|
||||
|
||||
return [legacy, next];
|
||||
}
|
||||
|
||||
/**
|
||||
* The residency-split gate and the `#new`->ClickHouse replication gate are
|
||||
* independent env vars. If split is on (run-ops runs are minted on the new DB) but the
|
||||
* constructed sources[] has no `"new"` source, every run-ops run is silently missing from
|
||||
* ClickHouse — under-counting all CH-fronted usage/cost/metrics aggregates with no
|
||||
* Postgres fallback. Couple the gates at boot: this misconfiguration must fail loudly
|
||||
* rather than ship a fleet-wide under-count.
|
||||
*/
|
||||
export class SplitReplicationMisconfiguredError extends Error {
|
||||
constructor() {
|
||||
super(
|
||||
'RUN_OPS_SPLIT_ENABLED is on but the runs-replication sources[] has no "new" source: ' +
|
||||
"run-ops runs on the new DB would not replicate to ClickHouse, under-counting every " +
|
||||
"ClickHouse-fronted aggregate. Enable the new replication source " +
|
||||
"(RUN_REPLICATION_NEW_ENABLED / RUN_REPLICATION_RUN_OPS_DATABASE_URL) or turn the split off."
|
||||
);
|
||||
this.name = "SplitReplicationMisconfiguredError";
|
||||
}
|
||||
}
|
||||
|
||||
export function assertReplicationCoversSplit(args: {
|
||||
splitEnabled: boolean;
|
||||
sources: RunsReplicationSource[];
|
||||
}): void {
|
||||
if (args.splitEnabled && !args.sources.some((s) => s.id === "new")) {
|
||||
throw new SplitReplicationMisconfiguredError();
|
||||
}
|
||||
}
|
||||
|
||||
function initializeRunsReplicationInstance() {
|
||||
const { DATABASE_URL } = process.env;
|
||||
invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
|
||||
|
||||
if (!env.RUN_REPLICATION_CLICKHOUSE_URL) {
|
||||
console.log("🗃️ Runs replication service not enabled");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("🗃️ Runs replication service enabled");
|
||||
|
||||
// Shared options for both the legacy-only and the multi-source constructions.
|
||||
// Excludes per-source identity (pgConnectionUrl/slotName/publicationName/sources).
|
||||
const baseReplicationOptions = {
|
||||
clickhouseFactory,
|
||||
serviceName: "runs-replication",
|
||||
redisOptions: {
|
||||
keyPrefix: "runs-replication:",
|
||||
port: env.RUN_REPLICATION_REDIS_PORT ?? undefined,
|
||||
host: env.RUN_REPLICATION_REDIS_HOST ?? undefined,
|
||||
username: env.RUN_REPLICATION_REDIS_USERNAME ?? undefined,
|
||||
password: env.RUN_REPLICATION_REDIS_PASSWORD ?? undefined,
|
||||
enableAutoPipelining: true,
|
||||
...(env.RUN_REPLICATION_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
|
||||
},
|
||||
maxFlushConcurrency: env.RUN_REPLICATION_MAX_FLUSH_CONCURRENCY,
|
||||
flushIntervalMs: env.RUN_REPLICATION_FLUSH_INTERVAL_MS,
|
||||
flushBatchSize: env.RUN_REPLICATION_FLUSH_BATCH_SIZE,
|
||||
leaderLockTimeoutMs: env.RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS,
|
||||
leaderLockExtendIntervalMs: env.RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS,
|
||||
leaderLockAcquireAdditionalTimeMs: env.RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS,
|
||||
leaderLockRetryIntervalMs: env.RUN_REPLICATION_LEADER_LOCK_RETRY_INTERVAL_MS,
|
||||
ackIntervalSeconds: env.RUN_REPLICATION_ACK_INTERVAL_SECONDS,
|
||||
logLevel: env.RUN_REPLICATION_LOG_LEVEL,
|
||||
waitForAsyncInsert: env.RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT === "1",
|
||||
tracer: provider.getTracer("runs-replication-service"),
|
||||
meter,
|
||||
insertMaxRetries: env.RUN_REPLICATION_INSERT_MAX_RETRIES,
|
||||
insertBaseDelayMs: env.RUN_REPLICATION_INSERT_BASE_DELAY_MS,
|
||||
insertMaxDelayMs: env.RUN_REPLICATION_INSERT_MAX_DELAY_MS,
|
||||
insertStrategy: env.RUN_REPLICATION_INSERT_STRATEGY,
|
||||
disablePayloadInsert: env.RUN_REPLICATION_DISABLE_PAYLOAD_INSERT === "1",
|
||||
disableErrorFingerprinting: env.RUN_REPLICATION_DISABLE_ERROR_FINGERPRINTING === "1",
|
||||
};
|
||||
|
||||
// Construct the SINGLE legacy source synchronously (the split gate has not resolved
|
||||
// yet at module-init time, and singleton(...) memoizes this synchronous return value).
|
||||
let service = new RunsReplicationService({
|
||||
...baseReplicationOptions,
|
||||
pgConnectionUrl: DATABASE_URL,
|
||||
slotName: env.RUN_REPLICATION_SLOT_NAME,
|
||||
publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
|
||||
// Explicit legacy source so the leader-lock key matches the id the status
|
||||
// route probes from the registry below.
|
||||
sources: [
|
||||
{
|
||||
id: "legacy",
|
||||
pgConnectionUrl: DATABASE_URL,
|
||||
slotName: env.RUN_REPLICATION_SLOT_NAME,
|
||||
publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
|
||||
originGeneration: env.RUN_REPLICATION_LEGACY_ORIGIN_GENERATION,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Register the live handle so the status route + lifecycle routes can find it.
|
||||
setRunsReplicationGlobal(service);
|
||||
setRunsReplicationConfiguredSources([
|
||||
{
|
||||
id: "legacy",
|
||||
slotName: env.RUN_REPLICATION_SLOT_NAME,
|
||||
originGeneration: env.RUN_REPLICATION_LEGACY_ORIGIN_GENERATION,
|
||||
},
|
||||
]);
|
||||
|
||||
if (env.RUN_REPLICATION_ENABLED === "1") {
|
||||
// Construct-after-gate: resolve the async split gate ONCE at boot, and
|
||||
// when both sources are enabled rebuild `service` with sources[] before starting.
|
||||
// The legacy-only instance above is never started in the dual path (no slot/lock
|
||||
// taken). runsReplicationService.server.ts is untouched. The create route also calls
|
||||
// setRunsReplicationGlobal — last-writer-wins is the existing contract.
|
||||
isSplitEnabled()
|
||||
.then(async (splitEnabled) => {
|
||||
const sources = buildReplicationSources({
|
||||
splitEnabled,
|
||||
legacyUrl: DATABASE_URL,
|
||||
newUrl: env.RUN_REPLICATION_RUN_OPS_DATABASE_URL,
|
||||
newSourceOverride: env.RUN_REPLICATION_NEW_ENABLED === "disabled" ? false : undefined,
|
||||
legacySlotName: env.RUN_REPLICATION_SLOT_NAME,
|
||||
legacyPublicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
|
||||
legacyOriginGeneration: env.RUN_REPLICATION_LEGACY_ORIGIN_GENERATION,
|
||||
newSlotName: env.RUN_REPLICATION_NEW_SLOT_NAME,
|
||||
newPublicationName: env.RUN_REPLICATION_NEW_PUBLICATION_NAME,
|
||||
newOriginGeneration: env.RUN_REPLICATION_NEW_ORIGIN_GENERATION,
|
||||
});
|
||||
|
||||
// Refuse to start replication if split is on but `#new` is not a source.
|
||||
assertReplicationCoversSplit({ splitEnabled, sources });
|
||||
|
||||
if (sources.length > 1) {
|
||||
// Release the bootstrap instance's eager replication client (Redis + Redlock)
|
||||
// before replacing it, or it leaks for the process lifetime. shutdown() is idempotent.
|
||||
await service.shutdown();
|
||||
// The scalar pgConnectionUrl/slotName/publicationName remain required on the
|
||||
// options type, but are ignored when sources[] is non-empty — the
|
||||
// service normalizes off sources. Pass the legacy scalars to satisfy the type.
|
||||
service = new RunsReplicationService({
|
||||
...baseReplicationOptions,
|
||||
pgConnectionUrl: DATABASE_URL,
|
||||
slotName: env.RUN_REPLICATION_SLOT_NAME,
|
||||
publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
|
||||
sources,
|
||||
});
|
||||
setRunsReplicationGlobal(service);
|
||||
setRunsReplicationConfiguredSources(
|
||||
sources.map((s) => ({
|
||||
id: s.id,
|
||||
slotName: s.slotName,
|
||||
originGeneration: s.originGeneration,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
return clickhouseFactory.isReady().then(() => service.start());
|
||||
})
|
||||
.then(() => console.log("🗃️ Runs replication service started"))
|
||||
.catch((error) => {
|
||||
if (error instanceof SplitReplicationMisconfiguredError) {
|
||||
// A silent ClickHouse under-count is worse than a crash — make it fatal.
|
||||
console.error("🚨 FATAL: run-ops split / ClickHouse replication misconfiguration", {
|
||||
error,
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
console.error("🗃️ Runs replication service failed to start", { error });
|
||||
});
|
||||
|
||||
// Closures over the `let` so SIGTERM/SIGINT hit whichever instance is live (NOT a
|
||||
// stale .bind() to the discarded legacy-only instance).
|
||||
signalsEmitter.on("SIGTERM", () => service.shutdown());
|
||||
signalsEmitter.on("SIGINT", () => service.shutdown());
|
||||
}
|
||||
|
||||
// Returns the legacy-only instance synchronously (singleton memoizes this). Lifecycle
|
||||
// routes read getRunsReplicationGlobal() first, so they get the live multi-source one.
|
||||
return service;
|
||||
}
|
||||
Reference in New Issue
Block a user