chore: import upstream snapshot with attribution

apps/sim/lib/tokenization/calculators.ts

@@ -0,0 +1,145 @@
+/**
+ * Cost calculation functions for tokenization
+ */
+
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { createTokenizationError } from '@/lib/tokenization/errors'
+import {
+  estimateInputTokens,
+  estimateOutputTokens,
+  estimateTokenCount,
+} from '@/lib/tokenization/estimators'
+import type {
+  CostBreakdown,
+  StreamingCostResult,
+  TokenizationInput,
+  TokenUsage,
+} from '@/lib/tokenization/types'
+import {
+  getProviderForTokenization,
+  logTokenizationDetails,
+  validateTokenizationInput,
+} from '@/lib/tokenization/utils'
+import { calculateCost } from '@/providers/utils'
+
+const logger = createLogger('TokenizationCalculators')
+
+/**
+ * Calculates cost estimate for streaming execution using token estimation
+ */
+export function calculateStreamingCost(
+  model: string,
+  inputText: string,
+  outputText: string,
+  systemPrompt?: string,
+  context?: string,
+  messages?: Array<{ role: string; content: string }>
+): StreamingCostResult {
+  try {
+    // Validate inputs
+    validateTokenizationInput(model, inputText, outputText)
+
+    const providerId = getProviderForTokenization(model)
+
+    // Estimate input tokens (combine all input sources)
+    const inputEstimate = estimateInputTokens(systemPrompt, context, messages, providerId)
+
+    // Add the main input text to the estimation
+    const mainInputEstimate = estimateTokenCount(inputText, providerId)
+    const totalPromptTokens = inputEstimate.count + mainInputEstimate.count
+
+    // Estimate output tokens
+    const outputEstimate = estimateOutputTokens(outputText, providerId)
+    const completionTokens = outputEstimate.count
+
+    // Calculate total tokens
+    const totalTokens = totalPromptTokens + completionTokens
+
+    // Create token usage object
+    const tokens: TokenUsage = {
+      input: totalPromptTokens,
+      output: completionTokens,
+      total: totalTokens,
+    }
+
+    // Calculate cost using provider pricing
+    const costResult = calculateCost(model, totalPromptTokens, completionTokens, false)
+
+    const cost: CostBreakdown = {
+      input: costResult.input,
+      output: costResult.output,
+      total: costResult.total,
+    }
+
+    const result: StreamingCostResult = {
+      tokens,
+      cost,
+      model,
+      provider: providerId,
+      method: 'tokenization',
+    }
+
+    logTokenizationDetails('Streaming cost calculation completed', {
+      model,
+      provider: providerId,
+      inputLength: inputText.length,
+      outputLength: outputText.length,
+      tokens,
+      cost,
+      method: 'tokenization',
+    })
+
+    return result
+  } catch (error) {
+    logger.error('Streaming cost calculation failed', {
+      model,
+      inputLength: inputText?.length || 0,
+      outputLength: outputText?.length || 0,
+      error: toError(error).message,
+    })
+
+    if (error instanceof Error && error.name === 'TokenizationError') {
+      throw error
+    }
+
+    throw createTokenizationError(
+      'CALCULATION_FAILED',
+      `Failed to calculate streaming cost: ${toError(error).message}`,
+      { model, inputLength: inputText?.length || 0, outputLength: outputText?.length || 0 }
+    )
+  }
+}
+
+/**
+ * Calculates cost for tokenization input object
+ */
+export function calculateTokenizationCost(input: TokenizationInput): StreamingCostResult {
+  return calculateStreamingCost(
+    input.model,
+    input.inputText,
+    input.outputText,
+    input.systemPrompt,
+    input.context,
+    input.messages
+  )
+}
+
+/**
+ * Creates a streaming cost result from existing provider response data
+ */
+export function createCostResultFromProviderData(
+  model: string,
+  providerTokens: TokenUsage,
+  providerCost: CostBreakdown
+): StreamingCostResult {
+  const providerId = getProviderForTokenization(model)
+
+  return {
+    tokens: providerTokens,
+    cost: providerCost,
+    model,
+    provider: providerId,
+    method: 'provider_response',
+  }
+}

apps/sim/lib/tokenization/constants.ts

@@ -0,0 +1,101 @@
+/**
+ * Configuration constants for tokenization functionality
+ */
+
+import type { ProviderTokenizationConfig } from '@/lib/tokenization/types'
+
+export const TOKENIZATION_CONFIG = {
+  providers: {
+    openai: {
+      avgCharsPerToken: 4,
+      confidence: 'high',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    'azure-openai': {
+      avgCharsPerToken: 4,
+      confidence: 'high',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    anthropic: {
+      avgCharsPerToken: 4.5,
+      confidence: 'high',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    'azure-anthropic': {
+      avgCharsPerToken: 4.5,
+      confidence: 'high',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    google: {
+      avgCharsPerToken: 5,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    deepseek: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    xai: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    cerebras: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    mistral: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    groq: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    sakana: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    nvidia: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    meta: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    zai: {
+      avgCharsPerToken: 4,
+      confidence: 'medium',
+      supportedMethods: ['heuristic', 'fallback'],
+    },
+    ollama: {
+      avgCharsPerToken: 4,
+      confidence: 'low',
+      supportedMethods: ['fallback'],
+    },
+  } satisfies Record<string, ProviderTokenizationConfig>,
+
+  fallback: {
+    avgCharsPerToken: 4,
+    confidence: 'low',
+    supportedMethods: ['fallback'],
+  } satisfies ProviderTokenizationConfig,
+
+  defaults: {
+    model: 'gpt-4o',
+    provider: 'openai',
+  },
+} as const
+
+export const LLM_BLOCK_TYPES = ['agent', 'router', 'evaluator'] as const
+
+export const MIN_TEXT_LENGTH_FOR_ESTIMATION = 1
+export const MAX_PREVIEW_LENGTH = 100

apps/sim/lib/tokenization/errors.ts

@@ -0,0 +1,23 @@
+/**
+ * Custom error classes for tokenization functionality
+ */
+
+export class TokenizationError extends Error {
+  public readonly code: 'INVALID_PROVIDER' | 'MISSING_TEXT' | 'CALCULATION_FAILED' | 'INVALID_MODEL'
+  public readonly details?: Record<string, unknown>
+
+  constructor(message: string, code: TokenizationError['code'], details?: Record<string, unknown>) {
+    super(message)
+    this.name = 'TokenizationError'
+    this.code = code
+    this.details = details
+  }
+}
+
+export function createTokenizationError(
+  code: TokenizationError['code'],
+  message: string,
+  details?: Record<string, unknown>
+): TokenizationError {
+  return new TokenizationError(message, code, details)
+}

apps/sim/lib/tokenization/estimators.ts

@@ -0,0 +1,341 @@
+/**
+ * Token estimation and accurate counting functions for different providers
+ */
+
+import { createLogger } from '@sim/logger'
+import { encodingForModel, type Tiktoken } from 'js-tiktoken'
+import { MIN_TEXT_LENGTH_FOR_ESTIMATION, TOKENIZATION_CONFIG } from '@/lib/tokenization/constants'
+import type { TokenEstimate } from '@/lib/tokenization/types'
+import { getProviderConfig } from '@/lib/tokenization/utils'
+
+const logger = createLogger('TokenizationEstimators')
+
+const encodingCache = new Map<string, Tiktoken>()
+
+/**
+ * Get or create a cached encoding for a model
+ */
+function getEncoding(modelName: string): Tiktoken {
+  if (encodingCache.has(modelName)) {
+    return encodingCache.get(modelName)!
+  }
+
+  try {
+    const encoding = encodingForModel(modelName as Parameters<typeof encodingForModel>[0])
+    encodingCache.set(modelName, encoding)
+    return encoding
+  } catch (error) {
+    logger.warn(`Failed to get encoding for model ${modelName}, falling back to cl100k_base`)
+    const encoding = encodingForModel('gpt-4')
+    encodingCache.set(modelName, encoding)
+    return encoding
+  }
+}
+
+if (typeof process !== 'undefined') {
+  process.on('beforeExit', () => {
+    clearEncodingCache()
+  })
+}
+
+/**
+ * Get accurate token count for text using tiktoken
+ * This is the exact count OpenAI's API will use
+ */
+export function getAccurateTokenCount(text: string, modelName = 'text-embedding-3-small'): number {
+  if (!text || text.length === 0) {
+    return 0
+  }
+
+  try {
+    const encoding = getEncoding(modelName)
+    const tokens = encoding.encode(text)
+    return tokens.length
+  } catch (error) {
+    logger.error('Error counting tokens with tiktoken:', error)
+    return Math.ceil(text.length / 4)
+  }
+}
+
+/**
+ * Get individual tokens as strings for visualization
+ * Returns an array of token strings that can be displayed with colors
+ */
+export function getTokenStrings(text: string, modelName = 'text-embedding-3-small'): string[] {
+  if (!text || text.length === 0) {
+    return []
+  }
+
+  try {
+    const encoding = getEncoding(modelName)
+    const tokenIds = encoding.encode(text)
+
+    const textChars = [...text]
+    const result: string[] = []
+    let prevCharCount = 0
+
+    for (let i = 0; i < tokenIds.length; i++) {
+      const decoded = encoding.decode(tokenIds.slice(0, i + 1))
+      const currentCharCount = [...decoded].length
+      const tokenCharCount = currentCharCount - prevCharCount
+
+      const tokenStr = textChars.slice(prevCharCount, prevCharCount + tokenCharCount).join('')
+      result.push(tokenStr)
+      prevCharCount = currentCharCount
+    }
+
+    return result
+  } catch (error) {
+    logger.error('Error getting token strings:', error)
+    return text.split(/(\s+)/).filter((s) => s.length > 0)
+  }
+}
+
+/**
+ * Truncate text to a maximum token count
+ * Useful for handling texts that exceed model limits
+ */
+export function truncateToTokenLimit(
+  text: string,
+  maxTokens: number,
+  modelName = 'text-embedding-3-small'
+): string {
+  if (!text || maxTokens <= 0) {
+    return ''
+  }
+
+  try {
+    const encoding = getEncoding(modelName)
+    const tokens = encoding.encode(text)
+
+    if (tokens.length <= maxTokens) {
+      return text
+    }
+
+    const truncatedTokens = tokens.slice(0, maxTokens)
+    const truncatedText = encoding.decode(truncatedTokens)
+
+    logger.warn(
+      `Truncated text from ${tokens.length} to ${maxTokens} tokens (${text.length} to ${truncatedText.length} chars)`
+    )
+
+    return truncatedText
+  } catch (error) {
+    logger.error('Error truncating text:', error)
+    const maxChars = maxTokens * 4
+    return text.slice(0, maxChars)
+  }
+}
+
+/**
+ * Batch texts by token count to stay within API limits
+ * Returns array of batches where each batch's total tokens <= maxTokensPerBatch
+ */
+export function batchByTokenLimit(
+  texts: string[],
+  maxTokensPerBatch: number,
+  modelName = 'text-embedding-3-small'
+): string[][] {
+  const batches: string[][] = []
+  let currentBatch: string[] = []
+  let currentTokenCount = 0
+
+  for (const text of texts) {
+    const tokenCount = getAccurateTokenCount(text, modelName)
+
+    if (tokenCount > maxTokensPerBatch) {
+      if (currentBatch.length > 0) {
+        batches.push(currentBatch)
+        currentBatch = []
+        currentTokenCount = 0
+      }
+
+      const truncated = truncateToTokenLimit(text, maxTokensPerBatch, modelName)
+      batches.push([truncated])
+      continue
+    }
+
+    if (currentBatch.length > 0 && currentTokenCount + tokenCount > maxTokensPerBatch) {
+      batches.push(currentBatch)
+      currentBatch = [text]
+      currentTokenCount = tokenCount
+    } else {
+      currentBatch.push(text)
+      currentTokenCount += tokenCount
+    }
+  }
+
+  if (currentBatch.length > 0) {
+    batches.push(currentBatch)
+  }
+
+  return batches
+}
+
+/**
+ * Clean up cached encodings (call when shutting down)
+ */
+export function clearEncodingCache(): void {
+  encodingCache.clear()
+  logger.info('Cleared tiktoken encoding cache')
+}
+
+/**
+ * Estimates token count for text using provider-specific heuristics
+ */
+export function estimateTokenCount(text: string, providerId?: string): TokenEstimate {
+  if (!text || text.length < MIN_TEXT_LENGTH_FOR_ESTIMATION) {
+    return {
+      count: 0,
+      confidence: 'high',
+      provider: providerId || 'unknown',
+      method: 'fallback',
+    }
+  }
+
+  const effectiveProviderId = providerId || TOKENIZATION_CONFIG.defaults.provider
+  const config = getProviderConfig(effectiveProviderId)
+
+  let estimatedTokens: number
+
+  switch (effectiveProviderId) {
+    case 'openai':
+    case 'azure-openai':
+      estimatedTokens = estimateOpenAITokens(text)
+      break
+    case 'anthropic':
+    case 'azure-anthropic':
+      estimatedTokens = estimateAnthropicTokens(text)
+      break
+    case 'google':
+      estimatedTokens = estimateGoogleTokens(text)
+      break
+    default:
+      estimatedTokens = estimateGenericTokens(text, config.avgCharsPerToken)
+  }
+
+  return {
+    count: Math.max(1, Math.round(estimatedTokens)),
+    confidence: config.confidence,
+    provider: effectiveProviderId,
+    method: 'heuristic',
+  }
+}
+
+/**
+ * OpenAI-specific token estimation using BPE characteristics
+ */
+function estimateOpenAITokens(text: string): number {
+  const words = text.trim().split(/\s+/)
+  let tokenCount = 0
+
+  for (const word of words) {
+    if (word.length === 0) continue
+
+    if (word.length <= 4) {
+      tokenCount += 1
+    } else if (word.length <= 8) {
+      tokenCount += Math.ceil(word.length / 4.5)
+    } else {
+      tokenCount += Math.ceil(word.length / 4)
+    }
+
+    const punctuationCount = (word.match(/[.,!?;:"'()[\]{}<>]/g) || []).length
+    tokenCount += punctuationCount * 0.5
+  }
+
+  const newlineCount = (text.match(/\n/g) || []).length
+  tokenCount += newlineCount * 0.5
+
+  return tokenCount
+}
+
+/**
+ * Anthropic Claude-specific token estimation
+ */
+function estimateAnthropicTokens(text: string): number {
+  const words = text.trim().split(/\s+/)
+  let tokenCount = 0
+
+  for (const word of words) {
+    if (word.length === 0) continue
+
+    if (word.length <= 4) {
+      tokenCount += 1
+    } else if (word.length <= 8) {
+      tokenCount += Math.ceil(word.length / 5)
+    } else {
+      tokenCount += Math.ceil(word.length / 4.5)
+    }
+  }
+
+  const newlineCount = (text.match(/\n/g) || []).length
+  tokenCount += newlineCount * 0.3
+
+  return tokenCount
+}
+
+/**
+ * Google Gemini-specific token estimation
+ */
+function estimateGoogleTokens(text: string): number {
+  const words = text.trim().split(/\s+/)
+  let tokenCount = 0
+
+  for (const word of words) {
+    if (word.length === 0) continue
+
+    if (word.length <= 5) {
+      tokenCount += 1
+    } else if (word.length <= 10) {
+      tokenCount += Math.ceil(word.length / 6)
+    } else {
+      tokenCount += Math.ceil(word.length / 5)
+    }
+  }
+
+  return tokenCount
+}
+
+/**
+ * Generic token estimation fallback
+ */
+function estimateGenericTokens(text: string, avgCharsPerToken: number): number {
+  const charCount = text.trim().length
+  return Math.ceil(charCount / avgCharsPerToken)
+}
+
+/**
+ * Estimates tokens for input content including context
+ */
+export function estimateInputTokens(
+  systemPrompt?: string,
+  context?: string,
+  messages?: Array<{ role: string; content: string }>,
+  providerId?: string
+): TokenEstimate {
+  let totalText = ''
+
+  if (systemPrompt) {
+    totalText += `${systemPrompt}\n`
+  }
+
+  if (context) {
+    totalText += `${context}\n`
+  }
+
+  if (messages) {
+    for (const message of messages) {
+      totalText += `${message.role}: ${message.content}\n`
+    }
+  }
+
+  return estimateTokenCount(totalText, providerId)
+}
+
+/**
+ * Estimates tokens for output content
+ */
+export function estimateOutputTokens(content: string, providerId?: string): TokenEstimate {
+  return estimateTokenCount(content, providerId)
+}

apps/sim/lib/tokenization/index.ts

@@ -0,0 +1,29 @@
+export {
+  calculateStreamingCost,
+  calculateTokenizationCost,
+  createCostResultFromProviderData,
+} from '@/lib/tokenization/calculators'
+export { LLM_BLOCK_TYPES, TOKENIZATION_CONFIG } from '@/lib/tokenization/constants'
+export { createTokenizationError, TokenizationError } from '@/lib/tokenization/errors'
+export {
+  batchByTokenLimit,
+  clearEncodingCache,
+  estimateInputTokens,
+  estimateOutputTokens,
+  estimateTokenCount,
+  getAccurateTokenCount,
+  truncateToTokenLimit,
+} from '@/lib/tokenization/estimators'
+export { processStreamingBlockLog, processStreamingBlockLogs } from '@/lib/tokenization/streaming'
+export {
+  createTextPreview,
+  extractTextContent,
+  formatTokenCount,
+  getProviderConfig,
+  getProviderForTokenization,
+  hasRealCostData,
+  hasRealTokenData,
+  isTokenizableBlockType,
+  logTokenizationDetails,
+  validateTokenizationInput,
+} from '@/lib/tokenization/utils'

apps/sim/lib/tokenization/streaming.ts

@@ -0,0 +1,149 @@
+/**
+ * Streaming-specific tokenization helpers
+ */
+
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { calculateStreamingCost } from '@/lib/tokenization/calculators'
+import { TOKENIZATION_CONFIG } from '@/lib/tokenization/constants'
+import {
+  extractTextContent,
+  hasRealCostData,
+  hasRealTokenData,
+  isTokenizableBlockType,
+  logTokenizationDetails,
+} from '@/lib/tokenization/utils'
+import type { BlockLog } from '@/executor/types'
+
+const logger = createLogger('StreamingTokenization')
+
+/**
+ * Processes a block log and adds tokenization data if needed
+ */
+export function processStreamingBlockLog(log: BlockLog, streamedContent: string): boolean {
+  // Check if this block should be tokenized
+  if (!isTokenizableBlockType(log.blockType)) {
+    return false
+  }
+
+  // Check if we already have meaningful token/cost data
+  if (hasRealTokenData(log.output?.tokens) && hasRealCostData(log.output?.cost)) {
+    return false
+  }
+
+  // Skip recalculation if cost was explicitly set by the billing layer (e.g. BYOK zero cost)
+  if (log.output?.cost?.pricing) {
+    return false
+  }
+
+  // Check if we have content to tokenize
+  if (!streamedContent?.trim()) {
+    return false
+  }
+
+  try {
+    // Determine model to use
+    const model = getModelForBlock(log)
+
+    // Prepare input text from log
+    const inputText = extractTextContent(log.input)
+
+    // Calculate streaming cost
+    const systemPrompt =
+      typeof log.input?.systemPrompt === 'string' ? log.input.systemPrompt : undefined
+    const context = typeof log.input?.context === 'string' ? log.input.context : undefined
+    const messages = Array.isArray(log.input?.messages)
+      ? (log.input.messages as Array<{ role: string; content: string }>)
+      : undefined
+    const result = calculateStreamingCost(
+      model,
+      inputText,
+      streamedContent,
+      systemPrompt,
+      context,
+      messages
+    )
+
+    // Update the log output with tokenization data
+    if (!log.output) {
+      log.output = {}
+    }
+
+    log.output.tokens = result.tokens
+    log.output.cost = result.cost
+    log.output.model = result.model
+
+    logTokenizationDetails(`Streaming tokenization completed for ${log.blockType}`, {
+      blockId: log.blockId,
+      blockType: log.blockType,
+      model: result.model,
+      provider: result.provider,
+      inputLength: inputText.length,
+      outputLength: streamedContent.length,
+      tokens: result.tokens,
+      cost: result.cost,
+      method: result.method,
+    })
+
+    return true
+  } catch (error) {
+    logger.error(`Streaming tokenization failed for block ${log.blockId}`, {
+      blockType: log.blockType,
+      error: toError(error).message,
+      contentLength: streamedContent?.length || 0,
+    })
+
+    // Don't throw - graceful degradation
+    return false
+  }
+}
+
+/**
+ * Determines the appropriate model for a block
+ */
+function getModelForBlock(log: BlockLog): string {
+  // Try to get model from output first
+  if (log.output?.model?.trim()) {
+    return log.output.model
+  }
+
+  // Try to get model from input
+  const inputModel = log.input?.model
+  if (typeof inputModel === 'string' && inputModel.trim()) {
+    return inputModel
+  }
+
+  // Use block type specific defaults
+  const blockType = log.blockType
+  if (blockType === 'agent' || blockType === 'router' || blockType === 'evaluator') {
+    return TOKENIZATION_CONFIG.defaults.model
+  }
+
+  // Final fallback
+  return TOKENIZATION_CONFIG.defaults.model
+}
+
+/**
+ * Processes multiple block logs for streaming tokenization
+ */
+export function processStreamingBlockLogs(
+  logs: BlockLog[],
+  streamedContentMap: Map<string, string>
+): number {
+  let processedCount = 0
+
+  for (const log of logs) {
+    const content = streamedContentMap.get(log.blockId)
+    if (content && processStreamingBlockLog(log, content)) {
+      processedCount++
+    }
+  }
+
+  logger.info(`Streaming tokenization summary`, {
+    totalLogs: logs.length,
+    processedBlocks: processedCount,
+    streamedBlocks: streamedContentMap.size,
+  })
+
+  return processedCount
+}

apps/sim/lib/tokenization/types.ts

@@ -0,0 +1,69 @@
+/**
+ * Type definitions for tokenization functionality
+ */
+
+export interface TokenEstimate {
+  /** Estimated number of tokens */
+  count: number
+  /** Confidence level of the estimation */
+  confidence: 'high' | 'medium' | 'low'
+  /** Provider used for estimation */
+  provider: string
+  /** Method used for estimation */
+  method: 'precise' | 'heuristic' | 'fallback'
+}
+
+export interface TokenUsage {
+  /** Number of input tokens */
+  input: number
+  /** Number of output tokens */
+  output: number
+  /** Total number of tokens */
+  total: number
+}
+
+export interface CostBreakdown {
+  /** Input cost in USD */
+  input: number
+  /** Output cost in USD */
+  output: number
+  /** Total cost in USD */
+  total: number
+}
+
+export interface StreamingCostResult {
+  /** Token usage breakdown */
+  tokens: TokenUsage
+  /** Cost breakdown */
+  cost: CostBreakdown
+  /** Model used for calculation */
+  model: string
+  /** Provider ID */
+  provider: string
+  /** Estimation method used */
+  method: 'tokenization' | 'provider_response'
+}
+
+export interface TokenizationInput {
+  /** Primary input text */
+  inputText: string
+  /** Generated output text */
+  outputText: string
+  /** Model identifier */
+  model: string
+  /** Optional system prompt */
+  systemPrompt?: string
+  /** Optional context */
+  context?: string
+  /** Optional message history */
+  messages?: Array<{ role: string; content: string }>
+}
+
+export interface ProviderTokenizationConfig {
+  /** Average characters per token for this provider */
+  avgCharsPerToken: number
+  /** Confidence level for this provider's estimation */
+  confidence: TokenEstimate['confidence']
+  /** Supported token estimation methods */
+  supportedMethods: TokenEstimate['method'][]
+}

apps/sim/lib/tokenization/utils.ts

@@ -0,0 +1,172 @@
+/**
+ * Utility functions for tokenization
+ */
+
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { truncate } from '@sim/utils/string'
+import {
+  LLM_BLOCK_TYPES,
+  MAX_PREVIEW_LENGTH,
+  TOKENIZATION_CONFIG,
+} from '@/lib/tokenization/constants'
+import { createTokenizationError } from '@/lib/tokenization/errors'
+import type { ProviderTokenizationConfig, TokenUsage } from '@/lib/tokenization/types'
+import type { BlockTokens } from '@/executor/types'
+import { getProviderFromModel } from '@/providers/utils'
+
+const logger = createLogger('TokenizationUtils')
+
+/**
+ * Gets tokenization configuration for a specific provider
+ */
+export function getProviderConfig(providerId: string): ProviderTokenizationConfig {
+  const config =
+    TOKENIZATION_CONFIG.providers[providerId as keyof typeof TOKENIZATION_CONFIG.providers]
+
+  if (!config) {
+    logger.debug(`No specific config for provider ${providerId}, using fallback`, { providerId })
+    return TOKENIZATION_CONFIG.fallback
+  }
+
+  return config
+}
+
+/**
+ * Extracts provider ID from model name
+ */
+export function getProviderForTokenization(model: string): string {
+  try {
+    return getProviderFromModel(model)
+  } catch (error) {
+    logger.warn(`Failed to get provider for model ${model}, using default`, {
+      model,
+      error: toError(error).message,
+    })
+    return TOKENIZATION_CONFIG.defaults.provider
+  }
+}
+
+/**
+ * Checks if a block type should be tokenized
+ */
+export function isTokenizableBlockType(blockType?: string): boolean {
+  if (!blockType) return false
+  return LLM_BLOCK_TYPES.includes(blockType as any)
+}
+
+/**
+ * Checks if tokens/cost data is meaningful (non-zero)
+ */
+export function hasRealTokenData(
+  tokens?: Pick<BlockTokens, 'total' | 'input' | 'output'>
+): boolean {
+  if (!tokens) return false
+  return (tokens.total ?? 0) > 0 || (tokens.input ?? 0) > 0 || (tokens.output ?? 0) > 0
+}
+
+/**
+ * Checks if cost data is meaningful (non-zero)
+ */
+export function hasRealCostData(cost?: {
+  total?: number
+  input?: number
+  output?: number
+}): boolean {
+  if (!cost) return false
+  return (cost.total || 0) > 0 || (cost.input || 0) > 0 || (cost.output || 0) > 0
+}
+
+/**
+ * Safely extracts text content from various input formats
+ */
+export function extractTextContent(input: unknown): string {
+  if (typeof input === 'string') {
+    return input.trim()
+  }
+
+  if (input && typeof input === 'object') {
+    try {
+      return JSON.stringify(input)
+    } catch (error) {
+      logger.warn('Failed to stringify input object', {
+        inputType: typeof input,
+        error: toError(error).message,
+      })
+      return ''
+    }
+  }
+
+  return String(input || '')
+}
+
+/**
+ * Creates a preview of text for logging (truncated)
+ */
+export function createTextPreview(text: string): string {
+  return truncate(text, MAX_PREVIEW_LENGTH)
+}
+
+/**
+ * Validates tokenization input
+ */
+export function validateTokenizationInput(
+  model: string,
+  inputText: string,
+  outputText: string
+): void {
+  if (!model?.trim()) {
+    throw createTokenizationError('INVALID_MODEL', 'Model is required for tokenization', { model })
+  }
+
+  if (!inputText?.trim() && !outputText?.trim()) {
+    throw createTokenizationError(
+      'MISSING_TEXT',
+      'Either input text or output text must be provided',
+      {
+        inputLength: inputText?.length || 0,
+        outputLength: outputText?.length || 0,
+      }
+    )
+  }
+}
+
+/**
+ * Formats token count for display
+ */
+export function formatTokenCount(count: number): string {
+  if (count === 0) return '0'
+  if (count < 1000) return count.toString()
+  if (count < 1000000) return `${(count / 1000).toFixed(1)}K`
+  return `${(count / 1000000).toFixed(1)}M`
+}
+
+/**
+ * Logs tokenization operation details
+ */
+export function logTokenizationDetails(
+  operation: string,
+  details: {
+    blockId?: string
+    blockType?: string
+    model?: string
+    provider?: string
+    inputLength?: number
+    outputLength?: number
+    tokens?: TokenUsage
+    cost?: { input?: number; output?: number; total?: number }
+    method?: string
+  }
+): void {
+  logger.info(`${operation}`, {
+    blockId: details.blockId,
+    blockType: details.blockType,
+    model: details.model,
+    provider: details.provider,
+    inputLength: details.inputLength,
+    outputLength: details.outputLength,
+    tokens: details.tokens,
+    cost: details.cost,
+    method: details.method,
+  })
+}