import type { Cache as UnkeyCache } from "@unkey/cache"; import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache"; import { createLRUMemoryStore } from "@internal/cache"; import { randomUUID } from "crypto"; import type { Redis } from "ioredis"; import type { EnvQueues, MarQSFairDequeueStrategy, MarQSKeyProducer } from "./types"; import seedrandom from "seedrandom"; import type { Tracer } from "@opentelemetry/api"; import { startSpan } from "../tracing.server"; export type FairDequeuingStrategyBiases = { /** * How much to bias towards environments with higher concurrency limits * 0 = no bias, 1 = full bias based on limit differences */ concurrencyLimitBias: number; /** * How much to bias towards environments with more available capacity * 0 = no bias, 1 = full bias based on available capacity */ availableCapacityBias: number; /** * Controls randomization of queue ordering within environments * 0 = strict age-based ordering (oldest first) * 1 = completely random ordering * Values between 0-1 blend between age-based and random ordering */ queueAgeRandomization: number; }; export type FairDequeuingStrategyOptions = { redis: Redis; keys: MarQSKeyProducer; defaultEnvConcurrency: number; parentQueueLimit: number; tracer: Tracer; seed?: string; /** * Configure biasing for environment shuffling * If not provided, no biasing will be applied (completely random shuffling) */ biases?: FairDequeuingStrategyBiases; reuseSnapshotCount?: number; maximumEnvCount?: number; /** * Maximum number of queues to process per environment * If not provided, all queues in an environment will be processed */ maximumQueuePerEnvCount?: number; }; type FairQueueConcurrency = { current: number; limit: number; reserve: number; }; type FairQueue = { id: string; age: number; org: string; env: string }; type FairQueueSnapshot = { id: string; envs: Record; queues: Array; }; type WeightedEnv = { envId: string; weight: number; }; type WeightedQueue = { queue: FairQueue; weight: number; }; const emptyFairQueueSnapshot: FairQueueSnapshot = { id: "empty", envs: {}, queues: [], }; const defaultBiases: FairDequeuingStrategyBiases = { concurrencyLimitBias: 0, availableCapacityBias: 0, queueAgeRandomization: 0, // Default to completely age-based ordering }; export class FairDequeuingStrategy implements MarQSFairDequeueStrategy { private _cache: UnkeyCache<{ concurrencyLimit: number; }>; private _rng: seedrandom.PRNG; private _reusedSnapshotForConsumer: Map< string, { snapshot: FairQueueSnapshot; reuseCount: number } > = new Map(); constructor(private options: FairDequeuingStrategyOptions) { const ctx = new DefaultStatefulContext(); const memory = createLRUMemoryStore(500); this._cache = createCache({ concurrencyLimit: new Namespace(ctx, { stores: [memory], fresh: 60_000, // The time in milliseconds that a value is considered fresh. Cache hits within this time will return the cached value. stale: 180_000, // The time in milliseconds that a value is considered stale. Cache hits within this time will return the cached value and trigger a background refresh. }), }); this._rng = seedrandom(options.seed); } async distributeFairQueuesFromParentQueue( parentQueue: string, consumerId: string ): Promise> { return await startSpan( this.options.tracer, "distributeFairQueuesFromParentQueue", async (span) => { span.setAttribute("consumer_id", consumerId); span.setAttribute("parent_queue", parentQueue); const snapshot = await this.#createQueueSnapshot(parentQueue, consumerId); span.setAttributes({ snapshot_env_count: Object.keys(snapshot.envs).length, snapshot_queue_count: snapshot.queues.length, }); const queues = snapshot.queues; if (queues.length === 0) { return []; } const envQueues = this.#shuffleQueuesByEnv(snapshot); span.setAttribute( "shuffled_queue_count", envQueues.reduce((sum, env) => sum + env.queues.length, 0) ); if (envQueues[0]?.queues[0]) { span.setAttribute("winning_env", envQueues[0].envId); span.setAttribute( "winning_org", this.options.keys.orgIdFromQueue(envQueues[0].queues[0]) ); } return envQueues; } ); } #shuffleQueuesByEnv(snapshot: FairQueueSnapshot): Array { const envs = Object.keys(snapshot.envs); const biases = this.options.biases ?? defaultBiases; if (biases.concurrencyLimitBias === 0 && biases.availableCapacityBias === 0) { const shuffledEnvs = this.#shuffle(envs); return this.#orderQueuesByEnvs(shuffledEnvs, snapshot); } // Find the maximum concurrency limit for normalization const maxLimit = Math.max(...envs.map((envId) => snapshot.envs[envId].concurrency.limit)); // Calculate weights for each environment const weightedEnvs: WeightedEnv[] = envs.map((envId) => { const env = snapshot.envs[envId]; // Start with base weight of 1 let weight = 1; // Add normalized concurrency limit bias if configured if (biases.concurrencyLimitBias > 0) { const normalizedLimit = env.concurrency.limit / maxLimit; // Square or cube the bias to make it more pronounced at higher values weight *= 1 + Math.pow(normalizedLimit * biases.concurrencyLimitBias, 2); } // Add available capacity bias if configured if (biases.availableCapacityBias > 0) { const usedCapacityPercentage = env.concurrency.current / env.concurrency.limit; const availableCapacityBonus = 1 - usedCapacityPercentage; // Square or cube the bias to make it more pronounced at higher values weight *= 1 + Math.pow(availableCapacityBonus * biases.availableCapacityBias, 2); } return { envId, weight }; }); const shuffledEnvs = this.#weightedShuffle(weightedEnvs); return this.#orderQueuesByEnvs(shuffledEnvs, snapshot); } #weightedShuffle(weightedItems: WeightedEnv[]): string[] { const totalWeight = weightedItems.reduce((sum, item) => sum + item.weight, 0); const result: string[] = []; const items = [...weightedItems]; while (items.length > 0) { let random = this._rng() * totalWeight; let index = 0; // Find item based on weighted random selection while (random > 0 && index < items.length) { random -= items[index].weight; index++; } index = Math.max(0, index - 1); // Add selected item to result and remove from items result.push(items[index].envId); items.splice(index, 1); } return result; } #orderQueuesByEnvs(envs: string[], snapshot: FairQueueSnapshot): Array { const queuesByEnv = snapshot.queues.reduce( (acc, queue) => { if (!acc[queue.env]) { acc[queue.env] = []; } acc[queue.env].push(queue); return acc; }, {} as Record> ); return envs.reduce((acc, envId) => { if (queuesByEnv[envId]) { // Get ordered queues for this env const orderedQueues = this.#weightedRandomQueueOrder(queuesByEnv[envId]); // Apply queue limit if maximumQueuePerEnvCount is set const limitedQueues = this.options.maximumQueuePerEnvCount ? orderedQueues.slice(0, this.options.maximumQueuePerEnvCount) : orderedQueues; // Only add the env if it has queues if (limitedQueues.length > 0) { acc.push({ envId, queues: limitedQueues.map((queue) => queue.id), }); } } return acc; }, [] as Array); } #weightedRandomQueueOrder(queues: FairQueue[]): FairQueue[] { if (queues.length <= 1) return queues; const biases = this.options.biases ?? defaultBiases; // When queueAgeRandomization is 0, use strict age-based ordering if (biases.queueAgeRandomization === 0) { return [...queues].sort((a, b) => b.age - a.age); } // Find the maximum age for normalization const maxAge = Math.max(...queues.map((q) => q.age)); // Calculate weights for each queue const weightedQueues: WeightedQueue[] = queues.map((queue) => { // Normalize age to be between 0 and 1 const normalizedAge = queue.age / maxAge; // Calculate weight: combine base weight with configurable age influence const baseWeight = 1; const weight = baseWeight + normalizedAge * biases.queueAgeRandomization; return { queue, weight }; }); // Perform weighted random selection for ordering const result: FairQueue[] = []; let remainingQueues = [...weightedQueues]; let totalWeight = remainingQueues.reduce((sum, wq) => sum + wq.weight, 0); while (remainingQueues.length > 0) { let random = this._rng() * totalWeight; let index = 0; // Find queue based on weighted random selection while (random > 0 && index < remainingQueues.length) { random -= remainingQueues[index].weight; index++; } index = Math.max(0, index - 1); // Add selected queue to result and remove from remaining result.push(remainingQueues[index].queue); totalWeight -= remainingQueues[index].weight; remainingQueues.splice(index, 1); } return result; } #shuffle(array: Array): Array { let currentIndex = array.length; let temporaryValue; let randomIndex; const newArray = [...array]; while (currentIndex !== 0) { randomIndex = Math.floor(this._rng() * currentIndex); currentIndex -= 1; temporaryValue = newArray[currentIndex]; newArray[currentIndex] = newArray[randomIndex]; newArray[randomIndex] = temporaryValue; } return newArray; } async #createQueueSnapshot(parentQueue: string, consumerId: string): Promise { return await startSpan(this.options.tracer, "createQueueSnapshot", async (span) => { span.setAttribute("consumer_id", consumerId); span.setAttribute("parent_queue", parentQueue); if ( typeof this.options.reuseSnapshotCount === "number" && this.options.reuseSnapshotCount > 0 ) { const key = `${parentQueue}:${consumerId}`; const reusedSnapshot = this._reusedSnapshotForConsumer.get(key); if (reusedSnapshot) { if (reusedSnapshot.reuseCount < this.options.reuseSnapshotCount) { span.setAttribute("reused_snapshot", true); this._reusedSnapshotForConsumer.set(key, { snapshot: reusedSnapshot.snapshot, reuseCount: reusedSnapshot.reuseCount + 1, }); return reusedSnapshot.snapshot; } else { this._reusedSnapshotForConsumer.delete(key); } } } span.setAttribute("reused_snapshot", false); const now = Date.now(); let queues = await this.#allChildQueuesByScore(parentQueue, consumerId, now); span.setAttribute("parent_queue_count", queues.length); if (queues.length === 0) { return emptyFairQueueSnapshot; } // Apply env selection if maximumEnvCount is specified let selectedEnvIds: Set; if (this.options.maximumEnvCount && this.options.maximumEnvCount > 0) { selectedEnvIds = this.#selectTopEnvs(queues, this.options.maximumEnvCount); // Filter queues to only include selected envs queues = queues.filter((queue) => selectedEnvIds.has(queue.env)); span.setAttribute("selected_env_count", selectedEnvIds.size); } span.setAttribute("selected_queue_count", queues.length); const envIds = new Set(); for (const queue of queues) { envIds.add(queue.env); } const envs = await Promise.all( Array.from(envIds).map(async (envId) => { return { id: envId, concurrency: await this.#getEnvConcurrency(envId) }; }) ); const envsAtFullConcurrency = envs.filter( (env) => env.concurrency.current >= env.concurrency.limit + env.concurrency.reserve ); const envIdsAtFullConcurrency = new Set(envsAtFullConcurrency.map((env) => env.id)); const envsSnapshot = envs.reduce( (acc, env) => { if (!envIdsAtFullConcurrency.has(env.id)) { acc[env.id] = env; } return acc; }, {} as Record ); span.setAttributes({ env_count: envs.length, envs_at_full_concurrency_count: envsAtFullConcurrency.length, }); const queuesSnapshot = queues.filter((queue) => !envIdsAtFullConcurrency.has(queue.env)); const snapshot = { id: randomUUID(), envs: envsSnapshot, queues: queuesSnapshot, }; if ( typeof this.options.reuseSnapshotCount === "number" && this.options.reuseSnapshotCount > 0 ) { this._reusedSnapshotForConsumer.set(`${parentQueue}:${consumerId}`, { snapshot, reuseCount: 0, }); } return snapshot; }); } #selectTopEnvs(queues: FairQueue[], maximumEnvCount: number): Set { // Group queues by env const queuesByEnv = queues.reduce( (acc, queue) => { if (!acc[queue.env]) { acc[queue.env] = []; } acc[queue.env].push(queue); return acc; }, {} as Record ); // Calculate average age for each env const envAverageAges = Object.entries(queuesByEnv).map(([envId, envQueues]) => { const averageAge = envQueues.reduce((sum, q) => sum + q.age, 0) / envQueues.length; return { envId, averageAge }; }); // Perform weighted shuffle based on average ages const maxAge = Math.max(...envAverageAges.map((e) => e.averageAge)); const weightedEnvs = envAverageAges.map((env) => ({ envId: env.envId, weight: env.averageAge / maxAge, // Normalize weights })); // Select top N envs using weighted shuffle const selectedEnvs = new Set(); let remainingEnvs = [...weightedEnvs]; let totalWeight = remainingEnvs.reduce((sum, env) => sum + env.weight, 0); while (selectedEnvs.size < maximumEnvCount && remainingEnvs.length > 0) { let random = this._rng() * totalWeight; let index = 0; while (random > 0 && index < remainingEnvs.length) { random -= remainingEnvs[index].weight; index++; } index = Math.max(0, index - 1); selectedEnvs.add(remainingEnvs[index].envId); totalWeight -= remainingEnvs[index].weight; remainingEnvs.splice(index, 1); } return selectedEnvs; } async #getEnvConcurrency(envId: string): Promise { return await startSpan(this.options.tracer, "getEnvConcurrency", async (span) => { span.setAttribute("env_id", envId); const [currentValue, limitValue, reserveValue] = await Promise.all([ this.#getEnvCurrentConcurrency(envId), this.#getEnvConcurrencyLimit(envId), this.#getEnvReserveConcurrency(envId), ]); span.setAttribute("current_value", currentValue); span.setAttribute("limit_value", limitValue); span.setAttribute("reserve_value", reserveValue); return { current: currentValue, limit: limitValue, reserve: reserveValue }; }); } async #allChildQueuesByScore( parentQueue: string, consumerId: string, now: number ): Promise> { return await startSpan(this.options.tracer, "allChildQueuesByScore", async (span) => { span.setAttribute("consumer_id", consumerId); span.setAttribute("parent_queue", parentQueue); const valuesWithScores = await this.options.redis.zrangebyscore( parentQueue, "-inf", now, "WITHSCORES", "LIMIT", 0, this.options.parentQueueLimit ); const result: Array = []; for (let i = 0; i < valuesWithScores.length; i += 2) { result.push({ id: valuesWithScores[i], age: now - Number(valuesWithScores[i + 1]), env: this.options.keys.envIdFromQueue(valuesWithScores[i]), org: this.options.keys.orgIdFromQueue(valuesWithScores[i]), }); } span.setAttribute("queue_count", result.length); if (result.length === this.options.parentQueueLimit) { span.setAttribute("parent_queue_limit_reached", true); } return result; }); } async #getEnvConcurrencyLimit(envId: string) { return await startSpan(this.options.tracer, "getEnvConcurrencyLimit", async (span) => { span.setAttribute("env_id", envId); const key = this.options.keys.envConcurrencyLimitKey(envId); const result = await this._cache.concurrencyLimit.swr(key, async () => { const value = await this.options.redis.get(key); if (!value) { return this.options.defaultEnvConcurrency; } return Number(value); }); return result.val ?? this.options.defaultEnvConcurrency; }); } async #getEnvCurrentConcurrency(envId: string) { return await startSpan(this.options.tracer, "getEnvCurrentConcurrency", async (span) => { span.setAttribute("env_id", envId); const key = this.options.keys.envCurrentConcurrencyKey(envId); const result = await this.options.redis.scard(key); span.setAttribute("current_value", result); return result; }); } async #getEnvReserveConcurrency(envId: string) { return await startSpan(this.options.tracer, "getEnvReserveConcurrency", async (span) => { span.setAttribute("env_id", envId); const key = this.options.keys.envReserveConcurrencyKey(envId); const result = await this.options.redis.scard(key); span.setAttribute("current_value", result); return result; }); } } export class NoopFairDequeuingStrategy implements MarQSFairDequeueStrategy { async distributeFairQueuesFromParentQueue( parentQueue: string, consumerId: string ): Promise> { return []; } }