chore: import upstream snapshot with attribution

apps/webapp/test/utils/marqs.ts

@@ -0,0 +1,116 @@
+import type { MarQSKeyProducer } from "~/v3/marqs/types";
+import { MarQSShortKeyProducer } from "~/v3/marqs/marqsKeyProducer.js";
+import type Redis from "ioredis";
+
+export function createKeyProducer(prefix: string): MarQSKeyProducer {
+  return new MarQSShortKeyProducer(prefix);
+}
+
+export type SetupQueueOptions = {
+  parentQueue: string;
+  redis: Redis;
+  score: number;
+  queueId: string;
+  orgId: string;
+  envId: string;
+  keyProducer: MarQSKeyProducer;
+};
+
+export type ConcurrencySetupOptions = {
+  keyProducer: MarQSKeyProducer;
+  redis: Redis;
+  orgId: string;
+  envId: string;
+  currentConcurrency?: number;
+  orgLimit?: number;
+  envLimit?: number;
+  isOrgDisabled?: boolean;
+};
+
+/**
+ * Adds a queue to Redis with the given parameters
+ */
+export async function setupQueue({
+  redis,
+  keyProducer,
+  parentQueue,
+  score,
+  queueId,
+  orgId,
+  envId,
+}: SetupQueueOptions) {
+  // Add the queue to the parent queue's sorted set
+  const queue = keyProducer.queueKey(orgId, envId, queueId);
+
+  await redis.zadd(parentQueue, score, queue);
+}
+
+type SetupConcurrencyOptions = {
+  redis: Redis;
+  keyProducer: MarQSKeyProducer;
+  env: { id: string; currentConcurrency: number; limit?: number; reserveConcurrency?: number };
+};
+
+/**
+ * Sets up concurrency-related Redis keys for orgs and envs
+ */
+export async function setupConcurrency({ redis, keyProducer, env }: SetupConcurrencyOptions) {
+  // Set env concurrency limit
+  if (typeof env.limit === "number") {
+    await redis.set(keyProducer.envConcurrencyLimitKey(env.id), env.limit.toString());
+  }
+
+  if (env.currentConcurrency > 0) {
+    // Set current concurrency by adding dummy members to the set
+    const envCurrentKey = keyProducer.envCurrentConcurrencyKey(env.id);
+
+    // Add dummy running job IDs to simulate current concurrency
+    const dummyJobs = Array.from(
+      { length: env.currentConcurrency },
+      (_, i) => `dummy-job-${i}-${Date.now()}`
+    );
+
+    await redis.sadd(envCurrentKey, ...dummyJobs);
+  }
+
+  if (env.reserveConcurrency && env.reserveConcurrency > 0) {
+    // Set reserved concurrency by adding dummy members to the set
+    const envReservedKey = keyProducer.envReserveConcurrencyKey(env.id);
+
+    // Add dummy reserved job IDs to simulate reserved concurrency
+    const dummyJobs = Array.from(
+      { length: env.reserveConcurrency },
+      (_, i) => `dummy-reserved-job-${i}-${Date.now()}`
+    );
+
+    await redis.sadd(envReservedKey, ...dummyJobs);
+  }
+}
+
+/**
+ * Calculates the standard deviation of a set of numbers.
+ * Standard deviation measures the amount of variation of a set of values from their mean.
+ * A low standard deviation indicates that the values tend to be close to the mean.
+ *
+ * @param values Array of numbers to calculate standard deviation for
+ * @returns The standard deviation of the values
+ */
+export function calculateStandardDeviation(values: number[]): number {
+  // If there are no values or only one value, the standard deviation is 0
+  if (values.length <= 1) {
+    return 0;
+  }
+
+  // Calculate the mean (average) of the values
+  const mean = values.reduce((sum, value) => sum + value, 0) / values.length;
+
+  // Calculate the sum of squared differences from the mean
+  const squaredDifferences = values.map((value) => Math.pow(value - mean, 2));
+  const sumOfSquaredDifferences = squaredDifferences.reduce((sum, value) => sum + value, 0);
+
+  // Calculate the variance (average of squared differences)
+  const variance = sumOfSquaredDifferences / (values.length - 1); // Using n-1 for sample standard deviation
+
+  // Standard deviation is the square root of the variance
+  return Math.sqrt(variance);
+}

apps/webapp/test/utils/replicationUtils.ts

@@ -0,0 +1,55 @@
+import { ClickHouse } from "@internal/clickhouse";
+import type { RedisOptions } from "@internal/redis";
+import type { PrismaClient } from "~/db.server";
+import { RunsReplicationService } from "~/services/runsReplicationService.server";
+import { TestReplicationClickhouseFactory } from "./testReplicationClickhouseFactory";
+import { afterEach } from "vitest";
+
+export async function setupClickhouseReplication({
+  prisma,
+  databaseUrl,
+  clickhouseUrl,
+  redisOptions,
+}: {
+  prisma: PrismaClient;
+  databaseUrl: string;
+  clickhouseUrl: string;
+  redisOptions: RedisOptions;
+}) {
+  await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
+
+  const clickhouse = new ClickHouse({
+    url: clickhouseUrl,
+    name: "runs-replication",
+    compression: {
+      request: true,
+    },
+  });
+
+  const runsReplicationService = new RunsReplicationService({
+    clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
+    pgConnectionUrl: databaseUrl,
+    serviceName: "runs-replication",
+    slotName: "task_runs_to_clickhouse_v1",
+    publicationName: "task_runs_to_clickhouse_v1_publication",
+    redisOptions,
+    maxFlushConcurrency: 1,
+    flushIntervalMs: 100,
+    flushBatchSize: 1,
+    leaderLockTimeoutMs: 5000,
+    leaderLockExtendIntervalMs: 1000,
+    ackIntervalSeconds: 5,
+  });
+
+  await runsReplicationService.start();
+
+  // Runs after each test in the current context
+  afterEach(async () => {
+    // Clean up resources here
+    await runsReplicationService.stop();
+  });
+
+  return {
+    clickhouse,
+  };
+}

apps/webapp/test/utils/streams.ts

@@ -0,0 +1,46 @@
+export async function convertResponseStreamToArray(response: Response): Promise<string[]> {
+  return convertReadableStreamToArray(response.body!.pipeThrough(new TextDecoderStream()));
+}
+
+export async function convertResponseSSEStreamToArray(response: Response): Promise<string[]> {
+  const parseSSEDataTransform = new TransformStream<string>({
+    async transform(chunk, controller) {
+      for (const line of chunk.split("\n")) {
+        if (line.startsWith("data:")) {
+          controller.enqueue(line.slice(6));
+        }
+      }
+    },
+  });
+
+  return convertReadableStreamToArray(
+    response.body!.pipeThrough(new TextDecoderStream()).pipeThrough(parseSSEDataTransform)
+  );
+}
+
+export async function convertReadableStreamToArray<T>(stream: ReadableStream<T>): Promise<T[]> {
+  const reader = stream.getReader();
+  const result: T[] = [];
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    result.push(value);
+  }
+
+  return result;
+}
+
+export function convertArrayToReadableStream<T>(values: T[]): ReadableStream<T> {
+  return new ReadableStream({
+    start(controller) {
+      try {
+        for (const value of values) {
+          controller.enqueue(value);
+        }
+      } finally {
+        controller.close();
+      }
+    },
+  });
+}

apps/webapp/test/utils/testReplicationClickhouseFactory.ts

@@ -0,0 +1,29 @@
+import type { ClickHouse } from "@internal/clickhouse";
+import { ClickhouseFactory, type ClientType } from "~/services/clickhouse/clickhouseFactory.server";
+import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
+
+const testReplicationRegistryStub = {
+  isLoaded: true,
+  isReady: Promise.resolve(),
+  get: () => null,
+} as unknown as OrganizationDataStoresRegistry;
+
+/**
+ * Routes all `replication` and `sessions_replication` clients to a single test ClickHouse;
+ * other client types use the real factory defaults.
+ */
+export class TestReplicationClickhouseFactory extends ClickhouseFactory {
+  constructor(private readonly replicationClient: ClickHouse) {
+    super(testReplicationRegistryStub);
+  }
+
+  override getClickhouseForOrganizationSync(
+    organizationId: string,
+    clientType: ClientType
+  ): ClickHouse {
+    if (clientType === "replication" || clientType === "sessions_replication") {
+      return this.replicationClient;
+    }
+    return super.getClickhouseForOrganizationSync(organizationId, clientType);
+  }
+}

apps/webapp/test/utils/timezones.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { isValidTimeZone } from "~/utils/timezones.server";
+
+describe("isValidTimeZone", () => {
+  // These are all zones a browser can report via
+  // Intl.DateTimeFormat().resolvedOptions().timeZone, but which are NOT in
+  // Intl.supportedValuesOf("timeZone"). Rejecting them left the user's stored
+  // timezone stale (their preference update would 400).
+  it.each(["UTC", "Etc/UTC", "GMT", "Asia/Kolkata"])(
+    "accepts %s even though it is not in supportedValuesOf",
+    (tz) => {
+      expect(Intl.supportedValuesOf("timeZone").includes(tz)).toBe(false);
+      expect(isValidTimeZone(tz)).toBe(true);
+    }
+  );
+
+  it.each(["Europe/London", "Europe/Moscow", "America/New_York", "Asia/Calcutta"])(
+    "accepts canonical zone %s",
+    (tz) => {
+      expect(isValidTimeZone(tz)).toBe(true);
+    }
+  );
+
+  it.each(["", "Not/AZone", "Mars/Phobos", "Europe/Nowhere", "12345"])(
+    "rejects invalid zone %s",
+    (tz) => {
+      expect(isValidTimeZone(tz)).toBe(false);
+    }
+  );
+});

apps/webapp/test/utils/tracing.ts

@@ -0,0 +1,89 @@
+import { context, propagation, trace } from "@opentelemetry/api";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+import { InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
+import {
+  MeterProvider,
+  InMemoryMetricExporter,
+  PeriodicExportingMetricReader,
+  AggregationTemporality,
+} from "@opentelemetry/sdk-metrics";
+
+export function createInMemoryTracing() {
+  // Webapp's vitest config uses `pool: "forks"` with `--no-file-parallelism`,
+  // so all test files in a shard share one process. globalThis persists
+  // across files even though vitest clears the module cache between them.
+  //
+  // OTel's `provider.register()` calls trace/context/propagation
+  // `setGlobal*` — and `setGlobal` no-ops (logs an error, returns false)
+  // when a global is already set. Two patterns hit that path:
+  // 1. `~/v3/tracer.server.ts` runs `provider.register()` via its
+  //    `singleton("opentelemetry", setupTelemetry)` — first test in the
+  //    shard to import that path sets the globals to webapp's tracer.
+  // 2. A subsequent test calls `createInMemoryTracing()` to swap in its
+  //    own in-memory provider. Without disabling first, register() is
+  //    a no-op — the test's provider receives spans via its
+  //    SimpleSpanProcessor (provider-scoped), but `trace.getActiveSpan()`
+  //    (used by code under test, e.g. sentryTraceContext.server.ts)
+  //    reads from the stale global context manager from step 1.
+  //
+  // Disable first, then register, so the test's provider always wins
+  // for both span recording and the global API. After the test, the
+  // next caller's createInMemoryTracing rotates again — no leakage.
+  trace.disable();
+  context.disable();
+  propagation.disable();
+
+  const exporter = new InMemorySpanExporter();
+  const provider = new NodeTracerProvider({
+    spanProcessors: [new SimpleSpanProcessor(exporter)],
+  });
+  provider.register();
+
+  // Use the provider's tracer so spans hit this exporter even when a global
+  // NodeTracerProvider was already registered (e.g. via tracer.server import chain).
+  const tracer = provider.getTracer("test-tracer");
+
+  return {
+    exporter,
+    tracer,
+  };
+}
+
+export function createInMemoryMetrics() {
+  // Initialize the metric exporter with cumulative temporality for easier testing
+  const metricExporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);
+
+  // Create a metric reader that exports frequently for testing
+  const metricReader = new PeriodicExportingMetricReader({
+    exporter: metricExporter,
+    exportIntervalMillis: 100, // Export frequently for tests
+  });
+
+  // Initialize the meter provider
+  const meterProvider = new MeterProvider({
+    readers: [metricReader],
+  });
+
+  // Retrieve a meter
+  const meter = meterProvider.getMeter("test-meter");
+
+  return {
+    metricExporter,
+    metricReader,
+    meterProvider,
+    meter,
+    // Helper to force collection and get metrics
+    async getMetrics() {
+      await metricReader.forceFlush();
+      return metricExporter.getMetrics();
+    },
+    // Helper to reset metrics between tests
+    reset() {
+      metricExporter.reset();
+    },
+    // Helper to shutdown the meter provider
+    async shutdown() {
+      await meterProvider.shutdown();
+    },
+  };
+}