chore: import upstream snapshot with attribution

open-sse/utils/cacheControlPolicy.ts

@@ -0,0 +1,374 @@
+/**
+ * Cache Control Policy
+ *
+ * Determines when to preserve client-side prompt caching headers (cache_control)
+ * vs. applying OmniRoute's own caching strategy.
+ *
+ * Client-side caching (e.g., Claude Code) should be preserved when:
+ * 1. Client is Claude Code or similar caching-aware client
+ * 2. Request will hit a deterministic target (single model or deterministic combo strategy)
+ * 3. Provider supports prompt caching (Anthropic, Alibaba Qwen, etc.)
+ */
+
+import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies";
+
+/**
+ * Cache control preservation modes
+ */
+export type CacheControlMode = "auto" | "always" | "never";
+
+/**
+ * Cache control settings from the database
+ */
+export interface CacheControlSettings {
+  alwaysPreserveClientCache?: CacheControlMode;
+}
+
+/**
+ * Cache metrics for tracking effectiveness
+ */
+export interface CacheControlMetrics {
+  // Totals
+  totalRequests: number;
+  requestsWithCacheControl: number;
+
+  // Token counts
+  totalInputTokens: number;
+  totalCachedTokens: number;
+  totalCacheCreationTokens: number;
+
+  // Savings
+  tokensSaved: number;
+  estimatedCostSaved: number;
+
+  // Breakdowns
+  byProvider: Record<
+    string,
+    {
+      requests: number;
+      inputTokens: number;
+      cachedTokens: number;
+      cacheCreationTokens: number;
+    }
+  >;
+  byStrategy: Record<
+    string,
+    {
+      requests: number;
+      inputTokens: number;
+      cachedTokens: number;
+      cacheCreationTokens: number;
+    }
+  >;
+
+  lastUpdated: string;
+}
+
+/**
+ * Routing strategies that are deterministic (same request → same provider)
+ */
+const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set(["priority", "cost-optimized"]);
+
+/**
+ * Providers that support prompt caching
+ */
+const CACHING_PROVIDERS = new Set([
+  "claude",
+  "anthropic",
+  "zai",
+  "qwen",
+  "deepseek",
+  // #3088 — Xiaomi MiMo honors OpenAI-format cache_control breakpoints. Without
+  // this entry, shouldPreserveCacheControl() returns false for Claude Code
+  // clients and filterToOpenAIFormat() strips cache_control, so Xiaomi never
+  // sees the cache hints and every request is a cache miss.
+  "xiaomi-mimo",
+  // #3955 — OpenAI / Codex / Azure-OpenAI use AUTOMATIC prefix caching: the longest
+  // matching prefix of a request is cached upstream WITHOUT any explicit cache_control
+  // markers. They must count as caching providers so the cache-aware compression guard
+  // preserves the cacheable prefix (system prompt / earliest messages) instead of
+  // rewriting it and forcing a cache miss. This also activates the intended
+  // `prompt_cache_key` cache-routing hint for OpenAI in chatCore.
+  "openai",
+  "codex",
+  "azure",
+  // #2069 — Alibaba DashScope's OpenAI-compatible endpoints (alibaba /
+  // alibaba-cn, upstream "alicode"/"alicode-intl") natively honor
+  // `cache_control: {type:"ephemeral"}` breakpoints. Without these entries
+  // shouldPreserveCacheControl() returns false for Claude Code clients and the
+  // OpenAI-format translator strips cache_control, so DashScope never sees the
+  // hints and every request is a cache miss.
+  "alibaba",
+  "alibaba-cn",
+]);
+
+/**
+ * Providers that honor EXPLICIT `cache_control` breakpoints carried inside an
+ * OpenAI-format request body (i.e. the markers must be passed THROUGH the
+ * Claude→OpenAI translation instead of stripped).
+ *
+ * This is a strict subset of CACHING_PROVIDERS and deliberately excludes
+ * `openai` / `codex` / `azure`: those use AUTOMATIC prefix caching (#3955) and
+ * do NOT accept explicit `cache_control` fields in the request — forwarding the
+ * markers there is meaningless at best and a 400 "unknown field" at worst, and
+ * it broke the chatCore "strips cache markers for non-Claude providers" test.
+ * Claude-format providers re-inject markers via prepareClaudeRequest, so they
+ * are not listed here either.
+ */
+const OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS = new Set([
+  // #2069 — DashScope OpenAI-compatible endpoints accept ephemeral breakpoints.
+  "alibaba",
+  "alibaba-cn",
+  // #3088 — Xiaomi MiMo honors OpenAI-format cache_control breakpoints.
+  "xiaomi-mimo",
+]);
+
+/**
+ * Whether `cache_control` markers should be PASSED THROUGH the OpenAI-format
+ * translation for this provider (vs. stripped). Used to gate the request-side
+ * passthrough so generic / implicit-cache OpenAI providers keep getting cleaned.
+ */
+export function providerHonorsOpenAIFormatCacheControl(
+  provider: string | null | undefined
+): boolean {
+  if (!provider) return false;
+  return OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS.has(provider.toLowerCase());
+}
+
+/**
+ * Detect if the client is Claude Code or another caching-aware client
+ */
+export function isClaudeCodeClient(userAgent: string | null | undefined): boolean {
+  if (!userAgent) return false;
+  const ua = userAgent.toLowerCase();
+
+  // Claude Code user agents
+  if (ua.includes("claude-code") || ua.includes("claude_code")) return true;
+  if (ua.includes("claude-cli/")) return true;
+  if (ua.includes("sdk-cli")) return true;
+  if (ua.includes("anthropic") && ua.includes("cli")) return true;
+
+  return false;
+}
+
+/**
+ * Check if a provider supports prompt caching
+ * Supports caching if:
+ * 1. Provider is in the known caching providers list, OR
+ * 2. Provider uses Claude protocol (detected via targetFormat)
+ */
+export function providerSupportsCaching(
+  provider: string | null | undefined,
+  targetFormat?: string | null
+): boolean {
+  if (!provider) return false;
+  if (CACHING_PROVIDERS.has(provider.toLowerCase())) return true;
+  // All Claude-protocol providers support prompt caching
+  if (targetFormat === "claude") return true;
+  return false;
+}
+
+/**
+ * Check if a routing strategy is deterministic
+ */
+export function isDeterministicStrategy(
+  strategy: RoutingStrategyValue | null | undefined
+): boolean {
+  if (!strategy) return false;
+  return DETERMINISTIC_STRATEGIES.has(strategy);
+}
+
+/**
+ * Determine if client-side cache_control headers should be preserved
+ *
+ * @param userAgent - User-Agent header from the request
+ * @param isCombo - Whether this is a combo model
+ * @param comboStrategy - The combo's routing strategy (if applicable)
+ * @param targetProvider - The target provider for the request
+ * @param settings - Cache control settings from database (optional)
+ * @returns true if cache_control should be preserved, false if OmniRoute should manage it
+ */
+export function shouldPreserveCacheControl({
+  userAgent,
+  isCombo,
+  comboStrategy,
+  targetProvider,
+  targetFormat,
+  settings,
+}: {
+  userAgent: string | null | undefined;
+  isCombo: boolean;
+  comboStrategy?: RoutingStrategyValue | null;
+  targetProvider: string | null | undefined;
+  targetFormat?: string | null;
+  settings?: CacheControlSettings;
+}): boolean {
+  // User override takes precedence
+  if (settings?.alwaysPreserveClientCache === "always") {
+    return true;
+  }
+  if (settings?.alwaysPreserveClientCache === "never") {
+    return false;
+  }
+
+  // Auto mode: use automatic detection (existing logic)
+  // Must be a caching-aware client
+  if (!isClaudeCodeClient(userAgent)) {
+    return false;
+  }
+
+  // Target provider must support caching
+  if (!providerSupportsCaching(targetProvider, targetFormat)) {
+    return false;
+  }
+
+  // Single model: always preserve (deterministic)
+  if (!isCombo) {
+    return true;
+  }
+
+  // Combo: only preserve if strategy is deterministic
+  return isDeterministicStrategy(comboStrategy);
+}
+
+/**
+ * Track cache control metrics for a request
+ */
+export function trackCacheMetrics({
+  preserved,
+  provider,
+  strategy,
+  metrics,
+  inputTokens,
+  cachedTokens,
+  cacheCreationTokens,
+}: {
+  preserved: boolean;
+  provider: string;
+  strategy: string | null | undefined;
+  metrics: CacheControlMetrics;
+  inputTokens?: number;
+  cachedTokens?: number;
+  cacheCreationTokens?: number;
+}): CacheControlMetrics {
+  const now = new Date().toISOString();
+
+  // Initialize metrics if empty
+  if (!metrics) {
+    metrics = {
+      totalRequests: 0,
+      requestsWithCacheControl: 0,
+      totalInputTokens: 0,
+      totalCachedTokens: 0,
+      totalCacheCreationTokens: 0,
+      tokensSaved: 0,
+      estimatedCostSaved: 0,
+      byProvider: {},
+      byStrategy: {},
+      lastUpdated: now,
+    };
+  }
+
+  // Increment total requests
+  metrics.totalRequests++;
+
+  // Track token counts
+  const input = inputTokens || 0;
+  const cached = cachedTokens || 0;
+  const creation = cacheCreationTokens || 0;
+
+  metrics.totalInputTokens += input;
+  metrics.totalCachedTokens += cached;
+  metrics.totalCacheCreationTokens += creation;
+
+  // Calculate tokens saved (cached tokens are reused, not charged)
+  if (cached > 0) {
+    metrics.tokensSaved += cached;
+  }
+
+  // Only track requests where cache_control was preserved
+  if (preserved) {
+    metrics.requestsWithCacheControl++;
+
+    // Initialize provider tracking
+    if (!metrics.byProvider[provider]) {
+      metrics.byProvider[provider] = {
+        requests: 0,
+        inputTokens: 0,
+        cachedTokens: 0,
+        cacheCreationTokens: 0,
+      };
+    }
+    metrics.byProvider[provider].requests++;
+    metrics.byProvider[provider].inputTokens += input;
+    metrics.byProvider[provider].cachedTokens += cached;
+    metrics.byProvider[provider].cacheCreationTokens += creation;
+
+    // Initialize strategy tracking
+    if (strategy && !metrics.byStrategy[strategy]) {
+      metrics.byStrategy[strategy] = {
+        requests: 0,
+        inputTokens: 0,
+        cachedTokens: 0,
+        cacheCreationTokens: 0,
+      };
+    }
+    if (strategy) {
+      metrics.byStrategy[strategy].requests++;
+      metrics.byStrategy[strategy].inputTokens += input;
+      metrics.byStrategy[strategy].cachedTokens += cached;
+      metrics.byStrategy[strategy].cacheCreationTokens += creation;
+    }
+  }
+
+  metrics.lastUpdated = now;
+  return metrics;
+}
+
+/**
+ * Record cache token usage and update metrics
+ */
+export function updateCacheTokenMetrics({
+  metrics,
+  provider,
+  strategy,
+  inputTokens,
+  cachedTokens,
+  cacheCreationTokens,
+  costSaved,
+}: {
+  metrics: CacheControlMetrics;
+  provider: string;
+  strategy: string | null | undefined;
+  inputTokens: number;
+  cachedTokens: number;
+  cacheCreationTokens: number;
+  costSaved?: number;
+}): CacheControlMetrics {
+  metrics.totalCachedTokens += cachedTokens;
+  metrics.totalCacheCreationTokens += cacheCreationTokens;
+  metrics.totalInputTokens += inputTokens;
+
+  // Cached tokens are reused (saved), creation tokens are new cache writes
+  metrics.tokensSaved += cachedTokens;
+  if (costSaved !== undefined) {
+    metrics.estimatedCostSaved += costSaved;
+  }
+
+  // Update provider tracking
+  if (metrics.byProvider[provider]) {
+    metrics.byProvider[provider].cachedTokens += cachedTokens;
+    metrics.byProvider[provider].cacheCreationTokens += cacheCreationTokens;
+    metrics.byProvider[provider].inputTokens += inputTokens;
+  }
+
+  // Update strategy tracking
+  if (strategy && metrics.byStrategy[strategy]) {
+    metrics.byStrategy[strategy].cachedTokens += cachedTokens;
+    metrics.byStrategy[strategy].cacheCreationTokens += cacheCreationTokens;
+    metrics.byStrategy[strategy].inputTokens += inputTokens;
+  }
+
+  metrics.lastUpdated = new Date().toISOString();
+  return metrics;
+}