wehub-resource-sync
131 lines
4.4 KiB
TypeScript
131 lines
4.4 KiB
TypeScript
/**
|
|
* Cache-aware strategy selection for AI model compression.
|
|
* Implements logic to detect caching context from request body and adjust strategies accordingly.
|
|
*
|
|
* @file cachingAware.ts
|
|
* @exports CachingContext, CacheAwareStrategy, detectCachingContext, getCacheAwareStrategy
|
|
*/
|
|
|
|
import { providerSupportsCaching } from "../../utils/cacheControlPolicy.ts";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
export interface CachingDetectionContext {
|
|
provider?: string | null;
|
|
targetFormat?: string | null;
|
|
model?: string | null;
|
|
}
|
|
|
|
export interface CachingContext {
|
|
hasCacheControl: boolean;
|
|
provider: string | null;
|
|
targetFormat: string | null;
|
|
isCachingProvider: boolean;
|
|
}
|
|
|
|
export interface CacheAwareStrategy {
|
|
strategy: string;
|
|
skipSystemPrompt: boolean;
|
|
deterministicOnly: boolean;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is JsonRecord {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function normalizeString(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function inferProviderFromModel(model: unknown): string | null {
|
|
const normalized = normalizeString(model);
|
|
if (!normalized) return null;
|
|
const slashIndex = normalized.indexOf("/");
|
|
if (slashIndex <= 0) return null;
|
|
return normalized.slice(0, slashIndex).toLowerCase();
|
|
}
|
|
|
|
function hasOwnCacheControl(value: unknown): boolean {
|
|
return isRecord(value) && value.cache_control !== undefined && value.cache_control !== null;
|
|
}
|
|
|
|
function arrayHasCacheControl(values: unknown): boolean {
|
|
return Array.isArray(values) && values.some((value) => hasCacheControl(value));
|
|
}
|
|
|
|
function hasCacheControl(value: unknown): boolean {
|
|
if (hasOwnCacheControl(value)) return true;
|
|
if (Array.isArray(value)) return value.some((item) => hasCacheControl(item));
|
|
if (!isRecord(value)) return false;
|
|
|
|
if (arrayHasCacheControl(value.system)) return true;
|
|
if (arrayHasCacheControl(value.tools)) return true;
|
|
if (arrayHasCacheControl(value.messages)) return true;
|
|
if (arrayHasCacheControl(value.input)) return true;
|
|
if (arrayHasCacheControl(value.contents)) return true;
|
|
|
|
if (isRecord(value.request) && arrayHasCacheControl(value.request.contents)) return true;
|
|
if (isRecord(value.content) || Array.isArray(value.content)) {
|
|
return hasCacheControl(value.content);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Detect the caching context from the request body.
|
|
*
|
|
* @param body - The request body to analyze
|
|
* @param context - Explicit provider/format/model metadata from the routing pipeline
|
|
* @returns A CachingContext object
|
|
*/
|
|
export function detectCachingContext(
|
|
body: unknown,
|
|
context: CachingDetectionContext = {}
|
|
): CachingContext {
|
|
const bodyRecord = isRecord(body) ? body : {};
|
|
const provider =
|
|
normalizeString(context.provider)?.toLowerCase() ??
|
|
inferProviderFromModel(context.model) ??
|
|
inferProviderFromModel(bodyRecord.model);
|
|
const targetFormat = normalizeString(context.targetFormat)?.toLowerCase() ?? null;
|
|
|
|
return {
|
|
hasCacheControl: hasCacheControl(body),
|
|
provider,
|
|
targetFormat,
|
|
isCachingProvider: providerSupportsCaching(provider, targetFormat),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get a cache-aware strategy based on the given strategy and caching context.
|
|
*
|
|
* @param strategy - The initial strategy (e.g., "aggressive", "ultra", etc.)
|
|
* @param ctx - The caching context
|
|
* @returns A CacheAwareStrategy object
|
|
*/
|
|
export function getCacheAwareStrategy(strategy: string, ctx: CachingContext): CacheAwareStrategy {
|
|
// #3955: a caching provider is enough on its own to protect the cacheable prefix.
|
|
// OpenAI / Codex (and other automatic-prefix-cache providers) carry NO explicit
|
|
// `cache_control` markers, yet the upstream still caches the longest matching prefix
|
|
// (system prompt / earliest messages). Gating on `hasCacheControl` skipped the guard
|
|
// for those providers, so a prefix-compressing mode rewrote the prefix → guaranteed
|
|
// cache miss. Treat `isCachingProvider` alone as sufficient; the explicit
|
|
// `cache_control` path is now a subset of this.
|
|
if (ctx.isCachingProvider) {
|
|
return {
|
|
strategy: ["aggressive", "ultra"].includes(strategy) ? "standard" : strategy,
|
|
skipSystemPrompt: true,
|
|
deterministicOnly: true,
|
|
};
|
|
}
|
|
|
|
// Return the original strategy with no modifications
|
|
return {
|
|
strategy,
|
|
skipSystemPrompt: false,
|
|
deterministicOnly: false,
|
|
};
|
|
}
|