wehub-resource-sync
d25d482dc2
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
102 lines
2.5 KiB
TypeScript
102 lines
2.5 KiB
TypeScript
/**
|
|
* Configuration constants for tokenization functionality
|
|
*/
|
|
|
|
import type { ProviderTokenizationConfig } from '@/lib/tokenization/types'
|
|
|
|
export const TOKENIZATION_CONFIG = {
|
|
providers: {
|
|
openai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
'azure-openai': {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
anthropic: {
|
|
avgCharsPerToken: 4.5,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
'azure-anthropic': {
|
|
avgCharsPerToken: 4.5,
|
|
confidence: 'high',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
google: {
|
|
avgCharsPerToken: 5,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
deepseek: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
xai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
cerebras: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
mistral: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
groq: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
sakana: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
nvidia: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
meta: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
zai: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'medium',
|
|
supportedMethods: ['heuristic', 'fallback'],
|
|
},
|
|
ollama: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'low',
|
|
supportedMethods: ['fallback'],
|
|
},
|
|
} satisfies Record<string, ProviderTokenizationConfig>,
|
|
|
|
fallback: {
|
|
avgCharsPerToken: 4,
|
|
confidence: 'low',
|
|
supportedMethods: ['fallback'],
|
|
} satisfies ProviderTokenizationConfig,
|
|
|
|
defaults: {
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
},
|
|
} as const
|
|
|
|
export const LLM_BLOCK_TYPES = ['agent', 'router', 'evaluator'] as const
|
|
|
|
export const MIN_TEXT_LENGTH_FOR_ESTIMATION = 1
|
|
export const MAX_PREVIEW_LENGTH = 100
|