+ +
+
+

1. Let's make a test request

+
The gateway supports 250+ models across 36 AI providers. Choose your provider and API + key below.
+
+
🐍 Python
+
📦 Node.js
+
🌀 cURL
+
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+

2. Create a routing config

+
Gateway configs allow you to route requests to different providers and models. You can load balance, set fallbacks, and configure automatic retries & timeouts. Learn more
+
+
Simple Config
+
Load Balancing
+
Fallbacks
+
Retries & Timeouts
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+

3. More Features to Explore

+
Discover advanced capabilities of the Portkey AI Gateway.
+
+ + + + + + + +

Agents

+

Seamlessly integrate with popular agent frameworks like Autogen, CrewAI, and LangChain.

+ + + + + + + +

Multi-modal AI

+

Handle vision, audio, and image generation requests across multiple providers.

+ + + + + +

Guardrails

+

Verify LLM inputs and outputs with 20+ pre-built checks or build your own.

+ + + + + + + + +

Supported Providers

+

Access 250+ models from 35+ providers including OpenAI, Anthropic, and Google.

+ +
+
+ +
+ + +
+
+ + + + +

Setup a Call

+

Get personalized support and learn how Portkey can be tailored to your needs.

+ Schedule Consultation +
+
+ + + + + +

Enterprise Features

+

Explore advanced features and see how Portkey can scale with your business.

+ View Enterprise Plan +
+
+ + + + +

Join Our Community

+

Connect with other developers, share ideas, and get help from the Portkey team.

+ Join Discord +
+
+
+
+ +
+
+

Real-time Logs

+
+ + +
+
+ + + + + + + + + + + + + + + + + + +
TimeMethodEndpointStatusDurationActions
+
+ Listening for logs... +
+
+
+
+ + + + + + + + + + + + + + diff --git a/src/services/conditionalRouter.ts b/src/services/conditionalRouter.ts new file mode 100644 index 0000000..ee272f3 --- /dev/null +++ b/src/services/conditionalRouter.ts @@ -0,0 +1,156 @@ +import { StrategyModes, Targets } from '../types/requestBody'; + +type Query = { + [key: string]: any; +}; + +interface RouterContext { + metadata?: Record; + params?: Record; + url?: { + pathname: string; + }; +} + +enum Operator { + // Comparison Operators + Equal = '$eq', + NotEqual = '$ne', + GreaterThan = '$gt', + GreaterThanOrEqual = '$gte', + LessThan = '$lt', + LessThanOrEqual = '$lte', + In = '$in', + NotIn = '$nin', + Regex = '$regex', + + // Logical Operators + And = '$and', + Or = '$or', +} + +export class ConditionalRouter { + private config: Targets; + private context: RouterContext; + + constructor(config: Targets, context: RouterContext) { + this.config = config; + this.context = context; + if (this.config.strategy?.mode !== StrategyModes.CONDITIONAL) { + throw new Error('Unsupported strategy mode'); + } + } + + resolveTarget(): Targets { + if (!this.config.strategy?.conditions) { + throw new Error('No conditions passed in the query router'); + } + + for (const condition of this.config.strategy.conditions) { + if (this.evaluateQuery(condition.query)) { + const targetName = condition.then; + return this.findTarget(targetName); + } + } + + // If no conditions matched and a default is specified, return the default target + if (this.config.strategy.default) { + return this.findTarget(this.config.strategy.default); + } + + throw new Error('Query router did not resolve to any valid target'); + } + + private evaluateQuery(query: Query): boolean { + for (const [key, value] of Object.entries(query)) { + if (key === Operator.Or && Array.isArray(value)) { + return value.some((subCondition: Query) => + this.evaluateQuery(subCondition) + ); + } + + if (key === Operator.And && Array.isArray(value)) { + return value.every((subCondition: Query) => + this.evaluateQuery(subCondition) + ); + } + + const contextValue = this.getContextValue(key); + + if (typeof value === 'object' && value !== null) { + if (!this.evaluateOperator(value, contextValue)) { + return false; + } + } else if (contextValue !== value) { + return false; + } + } + + return true; + } + + private evaluateOperator(operator: string, value: any): boolean { + for (const [op, compareValue] of Object.entries(operator)) { + switch (op) { + case Operator.Equal: + if (value !== compareValue) return false; + break; + case Operator.NotEqual: + if (value === compareValue) return false; + break; + case Operator.GreaterThan: + if (!(parseFloat(value) > parseFloat(compareValue))) return false; + break; + case Operator.GreaterThanOrEqual: + if (!(parseFloat(value) >= parseFloat(compareValue))) return false; + break; + case Operator.LessThan: + if (!(parseFloat(value) < parseFloat(compareValue))) return false; + break; + case Operator.LessThanOrEqual: + if (!(parseFloat(value) <= parseFloat(compareValue))) return false; + break; + case Operator.In: + if (!Array.isArray(compareValue) || !compareValue.includes(value)) + return false; + break; + case Operator.NotIn: + if (!Array.isArray(compareValue) || compareValue.includes(value)) + return false; + break; + case Operator.Regex: + try { + const regex = new RegExp(compareValue); + return regex.test(value); + } catch (e) { + return false; + } + default: + throw new Error( + `Unsupported operator used in the query router: ${op}` + ); + } + } + return true; + } + + private findTarget(name: string): Targets { + const index = + this.config.targets?.findIndex((target) => target.name === name) ?? -1; + if (index === -1) { + throw new Error(`Invalid target name found in the query router: ${name}`); + } + + return { + ...this.config.targets?.[index], + index, + }; + } + + private getContextValue(key: string): any { + const parts = key.split('.'); + let value: any = this.context; + value = value[parts[0]]?.[parts[1]]; + return value; + } +} diff --git a/src/services/realtimeLlmEventParser.ts b/src/services/realtimeLlmEventParser.ts new file mode 100644 index 0000000..12432c2 --- /dev/null +++ b/src/services/realtimeLlmEventParser.ts @@ -0,0 +1,165 @@ +import { Context } from 'hono'; +import { addBackgroundTask } from '../utils/misc'; + +export class RealtimeLlmEventParser { + private sessionState: any; + + constructor() { + this.sessionState = { + sessionDetails: null, + conversation: { + items: new Map(), + }, + responses: new Map(), + }; + } + + // Main entry point for processing events + handleEvent(c: Context, event: any, sessionOptions: any): void { + switch (event.type) { + case 'session.created': + this.handleSessionCreated(c, event, sessionOptions); + break; + case 'session.updated': + this.handleSessionUpdated(c, event, sessionOptions); + break; + case 'conversation.item.created': + this.handleConversationItemCreated(c, event); + break; + case 'conversation.item.deleted': + this.handleConversationItemDeleted(c, event); + break; + case 'response.done': + this.handleResponseDone(c, event, sessionOptions); + break; + case 'error': + this.handleError(c, event, sessionOptions); + break; + default: + break; + } + } + + // Handle `session.created` event + private handleSessionCreated( + c: Context, + data: any, + sessionOptions: any + ): void { + this.sessionState.sessionDetails = { ...data.session }; + const realtimeEventParser = c.get('realtimeEventParser'); + if (realtimeEventParser) { + addBackgroundTask( + c, + realtimeEventParser( + c, + sessionOptions, + {}, + { ...data.session }, + data.type + ) + ); + } + } + + // Handle `session.updated` event + private handleSessionUpdated( + c: Context, + data: any, + sessionOptions: any + ): void { + this.sessionState.sessionDetails = { ...data.session }; + const realtimeEventParser = c.get('realtimeEventParser'); + if (realtimeEventParser) { + addBackgroundTask( + c, + realtimeEventParser( + c, + sessionOptions, + {}, + { ...data.session }, + data.type + ) + ); + } + } + + // Conversation-specific handlers + private handleConversationItemCreated(c: Context, data: any): void { + const { item } = data; + this.sessionState.conversation.items.set(item.id, data); + } + + private handleConversationItemDeleted(c: Context, data: any): void { + this.sessionState.conversation.items.delete(data.item_id); + } + + private handleResponseDone(c: Context, data: any, sessionOptions: any): void { + const { response } = data; + this.sessionState.responses.set(response.id, response); + for (const item of response.output) { + const inProgressItem = this.sessionState.conversation.items.get(item.id); + this.sessionState.conversation.items.set(item.id, { + ...inProgressItem, + item, + }); + } + const realtimeEventParser = c.get('realtimeEventParser'); + if (realtimeEventParser) { + const itemSequence = this.rebuildConversationSequence( + this.sessionState.conversation.items + ); + addBackgroundTask( + c, + realtimeEventParser( + c, + sessionOptions, + { + conversation: { + items: this.getOrderedConversationItems(itemSequence).slice( + 0, + -1 + ), + }, + }, + data, + data.type + ) + ); + } + } + + private handleError(c: Context, data: any, sessionOptions: any): void { + const realtimeEventParser = c.get('realtimeEventParser'); + if (realtimeEventParser) { + addBackgroundTask( + c, + realtimeEventParser(c, sessionOptions, {}, data, data.type) + ); + } + } + + private rebuildConversationSequence(items: Map): string[] { + const orderedItemIds: string[] = []; + + // Find the first item (no previous_item_id) + let currentId: string | undefined = Array.from(items.values()).find( + (data) => data.previous_item_id === null + )?.item?.id; + + // Traverse through the chain using previous_item_id + while (currentId) { + orderedItemIds.push(currentId); + const nextItem = Array.from(items.values()).find( + (data) => data.previous_item_id === currentId + ); + currentId = nextItem?.item?.id; + } + + return orderedItemIds; + } + + private getOrderedConversationItems(sequence: string[]): any { + return sequence.map((id) => this.sessionState.conversation.items.get(id)!); + } +} diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts new file mode 100644 index 0000000..996a249 --- /dev/null +++ b/src/services/transformToProviderRequest.ts @@ -0,0 +1,292 @@ +import { GatewayError } from '../errors/GatewayError'; +import { AZURE_OPEN_AI, FIREWORKS_AI } from '../globals'; +import ProviderConfigs from '../providers'; +import { endpointStrings, ProviderConfig } from '../providers/types'; +import { Options, Params } from '../types/requestBody'; + +// TODO: Refactor this file to use the providerOptions object instead of the provider string + +/** + * Helper function to set a nested property in an object. + * + * @param obj - The object on which to set the property. + * @param path - The dot-separated path to the property. + * @param value - The value to set the property to. + */ +function setNestedProperty(obj: any, path: string, value: any) { + const parts = path.split('.'); + let current = obj; + for (let i = 0; i < parts.length - 1; i++) { + if (!current[parts[i]]) { + current[parts[i]] = {}; + } + current = current[parts[i]]; + } + current[parts[parts.length - 1]] = value; +} + +const getValue = ( + configParam: string, + params: Params, + paramConfig: any, + providerOptions: Options +) => { + let value = params[configParam as keyof typeof params]; + + // If a transformation is defined for this parameter, apply it + if (paramConfig.transform) { + value = paramConfig.transform(params, providerOptions); + } + + if ( + value === 'portkey-default' && + paramConfig && + paramConfig.default !== undefined + ) { + // Set the transformed parameter to the default value + value = paramConfig.default; + } + + // If a minimum is defined for this parameter and the value is less than this, set the value to the minimum + // Also, we should only do this comparison if value is of type 'number' + if ( + typeof value === 'number' && + paramConfig && + paramConfig.min !== undefined && + value < paramConfig.min + ) { + value = paramConfig.min; + } + + // If a maximum is defined for this parameter and the value is more than this, set the value to the maximum + // Also, we should only do this comparison if value is of type 'number' + else if ( + typeof value === 'number' && + paramConfig && + paramConfig.max !== undefined && + value > paramConfig.max + ) { + value = paramConfig.max; + } + + return value; +}; + +export const transformUsingProviderConfig = ( + providerConfig: ProviderConfig, + params: Params, + providerOptions: Options +) => { + const transformedRequest: { [key: string]: any } = {}; + + // For each parameter in the provider's configuration + for (const configParam in providerConfig) { + // Get the config for this parameter + let paramConfigs = providerConfig[configParam]; + if (!Array.isArray(paramConfigs)) { + paramConfigs = [paramConfigs]; + } + + for (const paramConfig of paramConfigs) { + // If the parameter is present in the incoming request body + if (configParam in params) { + // Get the value for this parameter + const value = getValue( + configParam, + params, + paramConfig, + providerOptions + ); + + // Set the transformed parameter to the validated value + setNestedProperty( + transformedRequest, + paramConfig?.param as string, + value + ); + } + // If the parameter is not present in the incoming request body but is required, set it to the default value + else if ( + paramConfig && + paramConfig.required && + paramConfig.default !== undefined + ) { + let value; + if (typeof paramConfig.default === 'function') { + value = paramConfig.default(params, providerOptions); + } else { + value = paramConfig.default; + } + // Set the transformed parameter to the default value + setNestedProperty(transformedRequest, paramConfig.param, value); + } + } + } + + return transformedRequest; +}; + +/** + * Transforms the request body to match the structure required by the AI provider. + * It also ensures the values for each parameter are within the minimum and maximum + * constraints defined in the provider's configuration. If a required parameter is missing, + * it assigns the default value from the provider's configuration. + * + * @param provider - The name of the AI provider. + * @param params - The parameters for the request. + * @param fn - The function to call on the AI provider. + * + * @returns The transformed request body. + * + * @throws {Error} If the provider is not supported. + */ +const transformToProviderRequestJSON = ( + provider: string, + params: Params, + fn: string, + providerOptions: Options +): { [key: string]: any } => { + // Get the configuration for the specified provider + let providerConfig = ProviderConfigs[provider]; + if (providerConfig.getConfig) { + providerConfig = providerConfig.getConfig({ params, providerOptions })[fn]; + } else { + providerConfig = providerConfig[fn]; + } + + if (!providerConfig) { + throw new GatewayError(`${fn} is not supported by ${provider}`); + } + + return transformUsingProviderConfig(providerConfig, params, providerOptions); +}; + +const transformToProviderRequestFormData = ( + provider: string, + params: Params, + fn: string, + providerOptions: Options +): FormData => { + let providerConfig = ProviderConfigs[provider]; + if (providerConfig.getConfig) { + providerConfig = providerConfig.getConfig({ params, providerOptions })[fn]; + } else { + providerConfig = providerConfig[fn]; + } + const formData = new FormData(); + for (const configParam in providerConfig) { + let paramConfigs = providerConfig[configParam]; + if (!Array.isArray(paramConfigs)) { + paramConfigs = [paramConfigs]; + } + for (const paramConfig of paramConfigs) { + if (configParam in params) { + const value = getValue( + configParam, + params, + paramConfig, + providerOptions + ); + + formData.append(paramConfig.param, value); + } else if ( + paramConfig && + paramConfig.required && + paramConfig.default !== undefined + ) { + let value; + if (typeof paramConfig.default === 'function') { + value = paramConfig.default(params); + } else { + value = paramConfig.default; + } + formData.append(paramConfig.param, value); + } + } + } + return formData; +}; + +const transformToProviderRequestBody = ( + provider: string, + requestBody: ReadableStream, + requestHeaders: Record, + providerOptions: Options, + fn: string +) => { + let providerConfig = ProviderConfigs[provider]; + if (providerConfig.getConfig) { + providerConfig = providerConfig.getConfig({ params: {}, providerOptions }); + } + + return providerConfig.requestTransforms[fn](requestBody, requestHeaders); +}; + +/** + * Transforms the request parameters to the format expected by the provider. + * + * @param {string} provider - The name of the provider (e.g., 'openai', 'anthropic'). + * @param {Params} params - The parameters for the request. + * @param {Params | FormData} inputParams - The original input parameters. + * @param {endpointStrings} fn - The function endpoint being called (e.g., 'complete', 'chatComplete'). + * @returns {Params | FormData} - The transformed request parameters. + */ +export const transformToProviderRequest = ( + provider: string, + params: Params, + requestBody: Params | FormData | ArrayBuffer | ReadableStream | ArrayBuffer, + fn: endpointStrings, + requestHeaders: Record, + providerOptions: Options +) => { + // this returns a ReadableStream + if (fn === 'uploadFile') { + return transformToProviderRequestBody( + provider, + requestBody as ReadableStream, + requestHeaders, + providerOptions, + fn + ); + } + + if ( + fn === 'createFinetune' && + [AZURE_OPEN_AI, FIREWORKS_AI].includes(provider) + ) { + return transformToProviderRequestBody( + provider, + requestBody as ReadableStream, + requestHeaders, + providerOptions, + fn + ); + } + + if (requestBody instanceof FormData || requestBody instanceof ArrayBuffer) + return requestBody; + + if (fn === 'proxy') { + return params; + } + + const providerAPIConfig = ProviderConfigs[provider].api; + if ( + providerAPIConfig.transformToFormData && + providerAPIConfig.transformToFormData({ gatewayRequestBody: params }) + ) + return transformToProviderRequestFormData( + provider, + params as Params, + fn, + providerOptions + ); + return transformToProviderRequestJSON( + provider, + params as Params, + fn, + providerOptions + ); +}; + +export default transformToProviderRequest; diff --git a/src/shared/services/cache/backends/cloudflareKV.ts b/src/shared/services/cache/backends/cloudflareKV.ts new file mode 100644 index 0000000..fdcb10b --- /dev/null +++ b/src/shared/services/cache/backends/cloudflareKV.ts @@ -0,0 +1,230 @@ +/** + * @file src/services/cache/backends/cloudflareKV.ts + * Cloudflare KV cache backend implementation + */ + +import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; + +// Using console.log for now to avoid build issues +const logger = { + debug: (msg: string, ...args: any[]) => + console.debug(`[CloudflareKVCache] ${msg}`, ...args), + info: (msg: string, ...args: any[]) => + console.info(`[CloudflareKVCache] ${msg}`, ...args), + warn: (msg: string, ...args: any[]) => + console.warn(`[CloudflareKVCache] ${msg}`, ...args), + error: (msg: string, ...args: any[]) => + console.error(`[CloudflareKVCache] ${msg}`, ...args), +}; + +// Cloudflare KV client interface +interface ICloudflareKVClient { + get(key: string): Promise; + set(key: string, value: string, options?: CacheOptions): Promise; + del(key: string): Promise; + keys(prefix: string): Promise; +} + +export class CloudflareKVCacheBackend implements CacheBackend { + private client: ICloudflareKVClient; + private dbName: string; + + private stats: CacheStats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + size: 0, + expired: 0, + }; + + constructor(client: ICloudflareKVClient, dbName: string) { + this.client = client; + this.dbName = dbName; + } + + private getFullKey(key: string, namespace?: string): string { + return namespace + ? `${this.dbName}:${namespace}:${key}` + : `${this.dbName}:default:${key}`; + } + + private serializeEntry(entry: CacheEntry): string { + return JSON.stringify(entry); + } + + private deserializeEntry(data: string): CacheEntry { + return JSON.parse(data); + } + + async get( + key: string, + namespace?: string + ): Promise | null> { + try { + const fullKey = this.getFullKey(key, namespace); + const data = await this.client.get(fullKey); + + if (!data) { + this.stats.misses++; + return null; + } + + const entry = this.deserializeEntry(data); + + this.stats.hits++; + return entry; + } catch (error) { + logger.error('Cloudflare KV get error:', error); + this.stats.misses++; + return null; + } + } + + async set( + key: string, + value: T, + options: CacheOptions = {} + ): Promise { + try { + const fullKey = this.getFullKey(key, options.namespace); + const now = Date.now(); + + const entry: CacheEntry = { + value, + createdAt: now, + expiresAt: options.ttl ? now + options.ttl : undefined, + metadata: options.metadata, + }; + + const serialized = this.serializeEntry(entry); + + this.client.set(fullKey, serialized, options); + + this.stats.sets++; + } catch (error) { + logger.error('Cloudflare KV set error:', error); + throw error; + } + } + + async delete(key: string, namespace?: string): Promise { + try { + const fullKey = this.getFullKey(key, namespace); + const deleted = await this.client.del(fullKey); + + if (deleted > 0) { + this.stats.deletes++; + return true; + } + + return false; + } catch (error) { + logger.error('Cloudflare KV delete error:', error); + return false; + } + } + + async clear(namespace?: string): Promise { + logger.debug('Cloudflare KV clear not implemented', namespace); + } + + async keys(namespace?: string): Promise { + try { + const prefix = namespace ? `cache:${namespace}:` : 'cache:default:'; + const fullKeys = await this.client.keys(prefix); + + return fullKeys.map((key) => key.substring(prefix.length)); + } catch (error) { + logger.error('Cloudflare KV keys error:', error); + return []; + } + } + + async getStats(namespace?: string): Promise { + try { + const prefix = namespace ? `cache:${namespace}:` : 'cache:default:'; + const keys = await this.client.keys(prefix); + + return { + ...this.stats, + size: keys.length, + }; + } catch (error) { + logger.error('Cloudflare KV getStats error:', error); + return { ...this.stats }; + } + } + + async has(key: string, namespace?: string): Promise { + logger.info('Cloudflare KV has not implemented', key, namespace); + return false; + } + + async cleanup(): Promise { + // Cloudflare KV handles TTL automatically, so this is mostly a no-op + // We could scan for entries with manual expiration and clean them up + logger.debug( + 'Cloudflare KV cleanup - TTL handled automatically by Cloudflare KV' + ); + } + + async close(): Promise { + logger.debug('Cloudflare KV close not implemented'); + } +} + +// Cloudflare KV client implementation +class CloudflareKVClient implements ICloudflareKVClient { + private KV: any; + + constructor(env: any, kvBindingName: string) { + this.KV = env[kvBindingName]; + } + + get = async (key: string): Promise => { + return await this.KV.get(key); + }; + + set = async ( + key: string, + value: string, + options?: CacheOptions + ): Promise => { + const kvOptions = { + expirationTtl: options?.ttl, + metadata: options?.metadata, + }; + try { + await this.KV.put(key, value, kvOptions); + return; + } catch (error) { + logger.error('Error setting key in Cloudflare KV:', error); + throw error; + } + }; + + del = async (key: string): Promise => { + try { + await this.KV.delete(key); + return 1; + } catch (error) { + logger.error('Error deleting key in Cloudflare KV:', error); + throw error; + } + }; + + keys = async (prefix: string): Promise => { + return await this.KV.list({ prefix }); + }; +} + +// Factory function to create Cloudflare KV backend +export function createCloudflareKVBackend( + env: any, + bindingName: string, + dbName: string +): CloudflareKVCacheBackend { + const client = new CloudflareKVClient(env, bindingName); + return new CloudflareKVCacheBackend(client, dbName); +} diff --git a/src/shared/services/cache/backends/file.ts b/src/shared/services/cache/backends/file.ts new file mode 100644 index 0000000..e517960 --- /dev/null +++ b/src/shared/services/cache/backends/file.ts @@ -0,0 +1,321 @@ +/** + * @file src/services/cache/backends/file.ts + * File-based cache backend implementation + */ + +import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; +import * as fs from 'fs/promises'; +import * as path from 'path'; + +// Using console.log for now to avoid build issues +const logger = { + debug: (msg: string, ...args: any[]) => + console.debug(`[FileCache] ${msg}`, ...args), + info: (msg: string, ...args: any[]) => + console.info(`[FileCache] ${msg}`, ...args), + warn: (msg: string, ...args: any[]) => + console.warn(`[FileCache] ${msg}`, ...args), + error: (msg: string, ...args: any[]) => + console.error(`[FileCache] ${msg}`, ...args), +}; + +interface FileCacheData { + [namespace: string]: { + [key: string]: CacheEntry; + }; +} + +export class FileCacheBackend implements CacheBackend { + private cacheFile: string; + private data: FileCacheData = {}; + private saveTimer?: NodeJS.Timeout; + private cleanupInterval?: NodeJS.Timeout; + private loaded: boolean = false; + private loadPromise: Promise; + private stats: CacheStats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + size: 0, + expired: 0, + }; + private saveInterval: number; + constructor( + dataDir: string = 'data', + fileName: string = 'cache.json', + saveIntervalMs: number = 1000, + cleanupIntervalMs: number = 60000 + ) { + this.cacheFile = path.join(process.cwd(), dataDir, fileName); + this.saveInterval = saveIntervalMs; + this.loadPromise = this.loadCache(); + this.loadPromise.then(() => { + this.startCleanup(cleanupIntervalMs); + }); + } + + // Ensure cache is loaded before any operation + private async ensureLoaded(): Promise { + if (!this.loaded) { + await this.loadPromise; + } + } + + private async ensureDataDir(): Promise { + const dir = path.dirname(this.cacheFile); + try { + await fs.mkdir(dir, { recursive: true }); + } catch (error) { + logger.error('Failed to create cache directory:', error); + } + } + + private async loadCache(): Promise { + try { + const content = await fs.readFile(this.cacheFile, 'utf-8'); + this.data = JSON.parse(content); + this.updateStats(); + logger.debug('Loaded cache from disk', this.cacheFile); + this.loaded = true; + } catch (error) { + // File doesn't exist or is invalid, start with empty cache + this.data = {}; + logger.debug('Starting with empty cache'); + } + } + + private async saveCache(): Promise { + try { + await this.ensureDataDir(); + await fs.writeFile(this.cacheFile, JSON.stringify(this.data, null, 2)); + logger.debug('Saved cache to disk'); + } catch (error) { + logger.error('Failed to save cache:', error); + } + } + + private scheduleSave(): void { + if (this.saveTimer) { + clearTimeout(this.saveTimer); + } + + this.saveTimer = setTimeout(() => { + this.saveCache(); + this.saveTimer = undefined; + }, this.saveInterval); + } + + private startCleanup(intervalMs: number): void { + this.cleanupInterval = setInterval(() => { + this.cleanup(); + }, intervalMs); + } + + private isExpired(entry: CacheEntry): boolean { + return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); + } + + private updateStats(): void { + let totalSize = 0; + let totalExpired = 0; + + for (const namespace of Object.values(this.data)) { + for (const entry of Object.values(namespace)) { + totalSize++; + if (this.isExpired(entry)) { + totalExpired++; + } + } + } + + this.stats.size = totalSize; + this.stats.expired = totalExpired; + } + + private getNamespaceData( + namespace: string = 'default' + ): Record { + if (!this.data[namespace]) { + this.data[namespace] = {}; + } + return this.data[namespace]; + } + + async get( + key: string, + namespace?: string + ): Promise | null> { + await this.ensureLoaded(); // Wait for load to complete + + const namespaceData = this.getNamespaceData(namespace); + const entry = namespaceData[key]; + + if (!entry) { + this.stats.misses++; + return null; + } + + if (this.isExpired(entry)) { + delete namespaceData[key]; + this.stats.expired++; + this.stats.misses++; + this.scheduleSave(); + return null; + } + + this.stats.hits++; + return entry as CacheEntry; + } + + async set( + key: string, + value: T, + options: CacheOptions = {} + ): Promise { + await this.ensureLoaded(); // Wait for load to complete + + const namespace = options.namespace || 'default'; + const namespaceData = this.getNamespaceData(namespace); + const now = Date.now(); + + const entry: CacheEntry = { + value, + createdAt: now, + expiresAt: options.ttl ? now + options.ttl : undefined, + metadata: options.metadata, + }; + + namespaceData[key] = entry; + this.stats.sets++; + this.updateStats(); + this.scheduleSave(); + } + + async delete(key: string, namespace?: string): Promise { + const namespaceData = this.getNamespaceData(namespace); + const existed = key in namespaceData; + + if (existed) { + delete namespaceData[key]; + this.stats.deletes++; + this.updateStats(); + this.scheduleSave(); + } + + return existed; + } + + async clear(namespace?: string): Promise { + if (namespace) { + const namespaceData = this.getNamespaceData(namespace); + const count = Object.keys(namespaceData).length; + this.data[namespace] = {}; + this.stats.deletes += count; + } else { + const totalCount = Object.values(this.data).reduce( + (sum, ns) => sum + Object.keys(ns).length, + 0 + ); + this.data = {}; + this.stats.deletes += totalCount; + } + + this.updateStats(); + this.scheduleSave(); + } + + async has(key: string, namespace?: string): Promise { + const namespaceData = this.getNamespaceData(namespace); + const entry = namespaceData[key]; + + if (!entry) return false; + + if (this.isExpired(entry)) { + delete namespaceData[key]; + this.stats.expired++; + this.scheduleSave(); + return false; + } + + return true; + } + + async keys(namespace?: string): Promise { + if (namespace) { + const namespaceData = this.getNamespaceData(namespace); + return Object.keys(namespaceData); + } + + const allKeys: string[] = []; + for (const namespaceData of Object.values(this.data)) { + allKeys.push(...Object.keys(namespaceData)); + } + return allKeys; + } + + async getStats(namespace?: string): Promise { + if (namespace) { + const namespaceData = this.getNamespaceData(namespace); + const keys = Object.keys(namespaceData); + let expired = 0; + + for (const key of keys) { + const entry = namespaceData[key]; + if (this.isExpired(entry)) { + expired++; + } + } + + return { + ...this.stats, + size: keys.length, + expired, + }; + } + + this.updateStats(); + return { ...this.stats }; + } + + async cleanup(): Promise { + let expiredCount = 0; + let hasChanges = false; + + for (const [, namespaceData] of Object.entries(this.data)) { + for (const [key, entry] of Object.entries(namespaceData)) { + if (this.isExpired(entry)) { + delete namespaceData[key]; + expiredCount++; + hasChanges = true; + } + } + } + + if (hasChanges) { + this.stats.expired += expiredCount; + this.updateStats(); + this.scheduleSave(); + logger.debug(`Cleaned up ${expiredCount} expired entries`); + } + } + + // Add method to check if ready + async waitForReady(): Promise { + await this.loadPromise; + } + + async close(): Promise { + if (this.saveTimer) { + clearTimeout(this.saveTimer); + await this.saveCache(); // Final save + } + + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = undefined; + } + + logger.debug('File cache backend closed'); + } +} diff --git a/src/shared/services/cache/backends/memory.ts b/src/shared/services/cache/backends/memory.ts new file mode 100644 index 0000000..f1e225d --- /dev/null +++ b/src/shared/services/cache/backends/memory.ts @@ -0,0 +1,220 @@ +/** + * @file src/services/cache/backends/memory.ts + * In-memory cache backend implementation + */ + +import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; +// Using console.log for now to avoid build issues +const logger = { + debug: (msg: string, ...args: any[]) => + console.debug(`[MemoryCache] ${msg}`, ...args), + info: (msg: string, ...args: any[]) => + console.info(`[MemoryCache] ${msg}`, ...args), + warn: (msg: string, ...args: any[]) => + console.warn(`[MemoryCache] ${msg}`, ...args), + error: (msg: string, ...args: any[]) => + console.error(`[MemoryCache] ${msg}`, ...args), +}; + +export class MemoryCacheBackend implements CacheBackend { + private cache = new Map(); + private stats: CacheStats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + size: 0, + expired: 0, + }; + private cleanupInterval?: NodeJS.Timeout; + private maxSize: number; + + constructor(maxSize: number = 10000, cleanupIntervalMs: number = 60000) { + this.maxSize = maxSize; + this.startCleanup(cleanupIntervalMs); + } + + private startCleanup(intervalMs: number): void { + this.cleanupInterval = setInterval(() => { + this.cleanup(); + }, intervalMs); + } + + private getFullKey(key: string, namespace?: string): string { + return namespace ? `${namespace}:${key}` : key; + } + + private isExpired(entry: CacheEntry): boolean { + return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); + } + + private evictIfNeeded(): void { + if (this.cache.size >= this.maxSize) { + // Simple LRU: remove oldest entries + const entries = Array.from(this.cache.entries()); + entries.sort((a, b) => a[1].createdAt - b[1].createdAt); + + const toRemove = Math.floor(this.maxSize * 0.1); // Remove 10% + for (let i = 0; i < toRemove && i < entries.length; i++) { + this.cache.delete(entries[i][0]); + } + + logger.debug(`Evicted ${toRemove} entries due to size limit`); + } + } + + async get( + key: string, + namespace?: string + ): Promise | null> { + const fullKey = this.getFullKey(key, namespace); + const entry = this.cache.get(fullKey); + + if (!entry) { + this.stats.misses++; + return null; + } + + if (this.isExpired(entry)) { + this.cache.delete(fullKey); + this.stats.expired++; + this.stats.misses++; + return null; + } + + this.stats.hits++; + return entry as CacheEntry; + } + + async set( + key: string, + value: T, + options: CacheOptions = {} + ): Promise { + const fullKey = this.getFullKey(key, options.namespace); + const now = Date.now(); + + const entry: CacheEntry = { + value, + createdAt: now, + expiresAt: options.ttl ? now + options.ttl : undefined, + metadata: options.metadata, + }; + + this.evictIfNeeded(); + this.cache.set(fullKey, entry); + this.stats.sets++; + this.stats.size = this.cache.size; + } + + async delete(key: string, namespace?: string): Promise { + const fullKey = this.getFullKey(key, namespace); + const deleted = this.cache.delete(fullKey); + + if (deleted) { + this.stats.deletes++; + this.stats.size = this.cache.size; + } + + return deleted; + } + + async clear(namespace?: string): Promise { + if (namespace) { + const prefix = `${namespace}:`; + const keysToDelete = Array.from(this.cache.keys()).filter((key) => + key.startsWith(prefix) + ); + + for (const key of keysToDelete) { + this.cache.delete(key); + } + + this.stats.deletes += keysToDelete.length; + } else { + this.stats.deletes += this.cache.size; + this.cache.clear(); + } + + this.stats.size = this.cache.size; + } + + async has(key: string, namespace?: string): Promise { + const fullKey = this.getFullKey(key, namespace); + const entry = this.cache.get(fullKey); + + if (!entry) return false; + + if (this.isExpired(entry)) { + this.cache.delete(fullKey); + this.stats.expired++; + return false; + } + + return true; + } + + async keys(namespace?: string): Promise { + const allKeys = Array.from(this.cache.keys()); + + if (namespace) { + const prefix = `${namespace}:`; + return allKeys + .filter((key) => key.startsWith(prefix)) + .map((key) => key.substring(prefix.length)); + } + + return allKeys; + } + + async getStats(namespace?: string): Promise { + if (namespace) { + const prefix = `${namespace}:`; + const namespaceKeys = Array.from(this.cache.keys()).filter((key) => + key.startsWith(prefix) + ); + + let expired = 0; + for (const key of namespaceKeys) { + const entry = this.cache.get(key); + if (entry && this.isExpired(entry)) { + expired++; + } + } + + return { + ...this.stats, + size: namespaceKeys.length, + expired, + }; + } + + return { ...this.stats }; + } + + async cleanup(): Promise { + let expiredCount = 0; + + for (const [key, entry] of this.cache.entries()) { + if (this.isExpired(entry)) { + this.cache.delete(key); + expiredCount++; + } + } + + if (expiredCount > 0) { + this.stats.expired += expiredCount; + this.stats.size = this.cache.size; + logger.debug(`Cleaned up ${expiredCount} expired entries`); + } + } + + async close(): Promise { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = undefined; + } + this.cache.clear(); + logger.debug('Memory cache backend closed'); + } +} diff --git a/src/shared/services/cache/backends/redis.ts b/src/shared/services/cache/backends/redis.ts new file mode 100644 index 0000000..64bd4db --- /dev/null +++ b/src/shared/services/cache/backends/redis.ts @@ -0,0 +1,246 @@ +/** + * @file src/services/cache/backends/redis.ts + * Redis cache backend implementation + */ +import Redis from 'ioredis'; + +import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; + +type RedisClient = Redis; + +// Using console.log for now to avoid build issues +const logger = { + debug: (msg: string, ...args: any[]) => + console.debug(`[RedisCache] ${msg}`, ...args), + info: (msg: string, ...args: any[]) => + console.info(`[RedisCache] ${msg}`, ...args), + warn: (msg: string, ...args: any[]) => + console.warn(`[RedisCache] ${msg}`, ...args), + error: (msg: string, ...args: any[]) => + console.error(`[RedisCache] ${msg}`, ...args), +}; + +export class RedisCacheBackend implements CacheBackend { + private client: RedisClient; + private dbName: string; + + private stats: CacheStats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + size: 0, + expired: 0, + }; + + constructor(client: RedisClient, dbName: string) { + this.client = client; + this.dbName = dbName; + } + + private serializeEntry(entry: CacheEntry): string { + return JSON.stringify(entry); + } + + private deserializeEntry(data: string): CacheEntry { + return JSON.parse(data); + } + + private isExpired(entry: CacheEntry): boolean { + return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); + } + + getFullKey(key: string, namespace?: string): string { + return namespace + ? `${this.dbName}:${namespace}:${key}` + : `${this.dbName}:default:${key}`; + } + + async get( + key: string, + namespace?: string + ): Promise | null> { + try { + const fullKey = this.getFullKey(key, namespace); + const data = await this.client.get(fullKey); + + if (!data) { + this.stats.misses++; + return null; + } + + const entry = this.deserializeEntry(data); + + // Double-check expiration (Redis TTL should handle this, but just in case) + if (this.isExpired(entry)) { + await this.client.del(fullKey); + this.stats.expired++; + this.stats.misses++; + return null; + } + + this.stats.hits++; + return entry; + } catch (error) { + logger.error('Redis get error:', error); + this.stats.misses++; + return null; + } + } + + async set( + key: string, + value: T, + options: CacheOptions = {} + ): Promise { + try { + const fullKey = this.getFullKey(key, options.namespace); + const now = Date.now(); + + const entry: CacheEntry = { + value, + createdAt: now, + expiresAt: options.ttl ? now + options.ttl : undefined, + metadata: options.metadata, + }; + + const serialized = this.serializeEntry(entry); + + if (options.ttl) { + // Set with TTL in seconds + const ttlSeconds = Math.ceil(options.ttl / 1000); + await this.client.set(fullKey, serialized, 'EX', ttlSeconds); + } else { + await this.client.set(fullKey, serialized); + } + + this.stats.sets++; + } catch (error) { + logger.error('Redis set error:', error); + throw error; + } + } + + async delete(key: string, namespace?: string): Promise { + try { + const fullKey = this.getFullKey(key, namespace); + const deleted = await this.client.del(fullKey); + + if (deleted > 0) { + this.stats.deletes++; + return true; + } + + return false; + } catch (error) { + logger.error('Redis delete error:', error); + return false; + } + } + + async clear(namespace?: string): Promise { + try { + const pattern = namespace + ? `${this.dbName}:${namespace}:*` + : `${this.dbName}:*`; + const keys = await this.client.keys(pattern); + + if (keys.length > 0) { + // Use single del call with spread operator for better performance + await this.client.del(...keys); + this.stats.deletes += keys.length; + } + } catch (error) { + logger.error('Redis clear error:', error); + throw error; + } + } + + async has(key: string, namespace?: string): Promise { + try { + const fullKey = this.getFullKey(key, namespace); + const exists = await this.client.exists(fullKey); + return exists > 0; + } catch (error) { + logger.error('Redis has error:', error); + return false; + } + } + + async keys(namespace?: string): Promise { + try { + const pattern = namespace + ? `${this.dbName}:${namespace}:*` + : `${this.dbName}:default:*`; + const fullKeys = await this.client.keys(pattern); + + // Extract the actual key part (remove the prefix) + const prefix = namespace + ? `${this.dbName}:${namespace}:` + : `${this.dbName}:default:`; + return fullKeys.map((key) => key.substring(prefix.length)); + } catch (error) { + logger.error('Redis keys error:', error); + return []; + } + } + + async getStats(namespace?: string): Promise { + try { + const pattern = namespace + ? `${this.dbName}:${namespace}:*` + : `${this.dbName}:*`; + const keys = await this.client.keys(pattern); + + return { + ...this.stats, + size: keys.length, + }; + } catch (error) { + logger.error('Redis getStats error:', error); + return { ...this.stats }; + } + } + + async script(mode: 'LOAD' | 'EXISTS', script: string): Promise { + return await this.client.script('LOAD', script); + } + + async evalsha(sha: string, keys: string[], args: string[]): Promise { + return await this.client.evalsha(sha, keys.length, ...keys, ...args); + } + + async cleanup(): Promise { + // Redis handles TTL automatically, so this is mostly a no-op + // We could scan for entries with manual expiration and clean them up + logger.debug('Redis cleanup - TTL handled automatically by Redis'); + } + + async close(): Promise { + try { + await this.client.quit(); + logger.debug('Redis cache backend closed'); + } catch (error) { + logger.error('Error closing Redis connection:', error); + } + } +} + +// Factory function to create Redis backend with ioredis +export function createRedisBackend( + redisUrl: string, + options?: any +): RedisCacheBackend { + // Extract dbName from options or use 'cache' as default + const dbName = options?.dbName || 'cache'; + + // Create ioredis client with URL and any additional options + // ioredis supports Redis URL format: redis://[username:password@]host[:port][/db] + const client = new Redis(redisUrl, { + ...options, + // Remove dbName from options as it's not an ioredis option + dbName: undefined, + }); + + return new RedisCacheBackend(client as RedisClient, dbName); +} diff --git a/src/shared/services/cache/index.ts b/src/shared/services/cache/index.ts new file mode 100644 index 0000000..8a5941c --- /dev/null +++ b/src/shared/services/cache/index.ts @@ -0,0 +1,490 @@ +/** + * @file src/services/cache/index.ts + * Unified cache service with pluggable backends + */ + +import { + CacheBackend, + CacheEntry, + CacheOptions, + CacheStats, + CacheConfig, +} from './types'; +import { MemoryCacheBackend } from './backends/memory'; +import { FileCacheBackend } from './backends/file'; +import { createRedisBackend } from './backends/redis'; +import { createCloudflareKVBackend } from './backends/cloudflareKV'; +// Using console.log for now to avoid build issues +const logger = { + debug: (msg: string, ...args: any[]) => + console.debug(`[CacheService] ${msg}`, ...args), + info: (msg: string, ...args: any[]) => + console.info(`[CacheService] ${msg}`, ...args), + warn: (msg: string, ...args: any[]) => + console.warn(`[CacheService] ${msg}`, ...args), + error: (msg: string, ...args: any[]) => + console.error(`[CacheService] ${msg}`, ...args), +}; + +const MS = { + '1_MINUTE': 1 * 60 * 1000, + '5_MINUTES': 5 * 60 * 1000, + '10_MINUTES': 10 * 60 * 1000, + '30_MINUTES': 30 * 60 * 1000, + '1_HOUR': 60 * 60 * 1000, + '6_HOURS': 6 * 60 * 60 * 1000, + '12_HOURS': 12 * 60 * 60 * 1000, + '1_DAY': 24 * 60 * 60 * 1000, + '7_DAYS': 7 * 24 * 60 * 60 * 1000, + '30_DAYS': 30 * 24 * 60 * 60 * 1000, +}; + +export class CacheService { + private backend: CacheBackend; + private defaultTtl?: number; + + constructor(config: CacheConfig) { + this.defaultTtl = config.defaultTtl; + this.backend = this.createBackend(config); + } + + private createBackend(config: CacheConfig): CacheBackend { + switch (config.backend) { + case 'memory': + return new MemoryCacheBackend(config.maxSize, config.cleanupInterval); + + case 'file': + return new FileCacheBackend( + config.dataDir, + config.fileName, + config.saveInterval, + config.cleanupInterval + ); + + case 'redis': + if (!config.redisUrl) { + throw new Error('Redis URL is required for Redis backend'); + } + return createRedisBackend(config.redisUrl, { + ...config.redisOptions, + dbName: config.dbName || 'cache', + }); + + case 'cloudflareKV': + if (!config.kvBindingName || !config.dbName) { + throw new Error( + 'Cloudflare KV binding name and db name are required for Cloudflare KV backend' + ); + } + return createCloudflareKVBackend( + config.env, + config.kvBindingName, + config.dbName + ); + + default: + throw new Error(`Unsupported cache backend: ${config.backend}`); + } + } + + /** + * Get a value from the cache + */ + async get(key: string, namespace?: string): Promise { + const entry = await this.backend.get(key, namespace); + return entry ? entry.value : null; + } + + /** + * Get the full cache entry (with metadata) + */ + async getEntry( + key: string, + namespace?: string + ): Promise | null> { + return this.backend.get(key, namespace); + } + + /** + * Set a value in the cache + */ + async set( + key: string, + value: T, + options: CacheOptions = {} + ): Promise { + const finalOptions = { + ...options, + ttl: options.ttl ?? this.defaultTtl, + }; + + await this.backend.set(key, value, finalOptions); + } + + /** + * Set a value with TTL in seconds (convenience method) + */ + async setWithTtl( + key: string, + value: T, + ttlSeconds: number, + namespace?: string + ): Promise { + await this.set(key, value, { + ttl: ttlSeconds * 1000, + namespace, + }); + } + + /** + * Delete a value from the cache + */ + async delete(key: string, namespace?: string): Promise { + return this.backend.delete(key, namespace); + } + + /** + * Check if a key exists in the cache + */ + async has(key: string, namespace?: string): Promise { + return this.backend.has(key, namespace); + } + + /** + * Get all keys in a namespace + */ + async keys(namespace?: string): Promise { + return this.backend.keys(namespace); + } + + /** + * Clear all entries in a namespace (or all entries if no namespace) + */ + async clear(namespace?: string): Promise { + await this.backend.clear(namespace); + } + + /** + * Get cache statistics + */ + async getStats(namespace?: string): Promise { + return this.backend.getStats(namespace); + } + + /** + * Manually trigger cleanup of expired entries + */ + async cleanup(): Promise { + await this.backend.cleanup(); + } + + /** + * Wait for the backend to be ready + */ + async waitForReady(): Promise { + if ('waitForReady' in this.backend) { + await (this.backend as any).waitForReady(); + } + } + + /** + * Close the cache and cleanup resources + */ + async close(): Promise { + await this.backend.close(); + } + + /** + * Get or set pattern - get value, or compute and cache it if not found + */ + async getOrSet( + key: string, + factory: () => Promise | T, + options: CacheOptions = {} + ): Promise { + const existing = await this.get(key, options.namespace); + if (existing !== null) { + return existing; + } + + const value = await factory(); + await this.set(key, value, options); + return value; + } + + /** + * Increment a numeric value (atomic operation for supported backends) + */ + async increment( + key: string, + delta: number = 1, + options: CacheOptions = {} + ): Promise { + // For backends that don't support atomic increment, we simulate it + const current = (await this.get(key, options.namespace)) || 0; + const newValue = current + delta; + await this.set(key, newValue, options); + return newValue; + } + + /** + * Set multiple values at once + */ + async setMany( + entries: Array<{ key: string; value: T; options?: CacheOptions }>, + defaultOptions: CacheOptions = {} + ): Promise { + const promises = entries.map(({ key, value, options }) => + this.set(key, value, { ...defaultOptions, ...options }) + ); + await Promise.all(promises); + } + + /** + * Get multiple values at once + */ + async getMany( + keys: string[], + namespace?: string + ): Promise> { + const promises = keys.map(async (key) => ({ + key, + value: await this.get(key, namespace), + })); + return Promise.all(promises); + } + + getClient(): CacheBackend { + return this.backend; + } +} + +// Default cache instances for different use cases +let defaultCache: CacheService | null = null; +let tokenCache: CacheService | null = null; +let sessionCache: CacheService | null = null; +let configCache: CacheService | null = null; +let oauthStore: CacheService | null = null; +let mcpServersCache: CacheService | null = null; +let apiRateLimiterCache: CacheService | null = null; +/** + * Get or create the default cache instance + */ +export function getDefaultCache(): CacheService { + if (!defaultCache) { + throw new Error('Default cache instance not found'); + } + return defaultCache; +} + +/** + * Get or create the token cache instance + */ +export function getTokenCache(): CacheService { + if (!tokenCache) { + throw new Error('Token cache instance not found'); + } + return tokenCache; +} + +/** + * Get or create the session cache instance + */ +export function getSessionCache(): CacheService { + if (!sessionCache) { + throw new Error('Session cache instance not found'); + } + return sessionCache; +} + +/** + * Get or create the token introspection cache instance + */ +export function getTokenIntrospectionCache(): CacheService { + // Use the same cache as tokens, just different namespace + return getTokenCache(); +} + +/** + * Get or create the config cache instance + */ +export function getConfigCache(): CacheService { + if (!configCache) { + throw new Error('Config cache instance not found'); + } + return configCache; +} + +/** + * Get or create the oauth store cache instance + */ +export function getOauthStore(): CacheService { + if (!oauthStore) { + throw new Error('Oauth store cache instance not found'); + } + return oauthStore; +} + +export function getMcpServersCache(): CacheService { + if (!mcpServersCache) { + throw new Error('Mcp servers cache instance not found'); + } + return mcpServersCache; +} + +/** + * Initialize cache with custom configuration + */ +export function initializeCache(config: CacheConfig): CacheService { + return new CacheService(config); +} + +export async function createCacheBackendsLocal(): Promise { + defaultCache = new CacheService({ + backend: 'memory', + defaultTtl: MS['5_MINUTES'], + cleanupInterval: MS['5_MINUTES'], + maxSize: 1000, + }); + + tokenCache = new CacheService({ + backend: 'memory', + defaultTtl: MS['5_MINUTES'], + saveInterval: 1000, // 1 second + cleanupInterval: MS['5_MINUTES'], + maxSize: 1000, + }); + + sessionCache = new CacheService({ + backend: 'file', + dataDir: 'data', + fileName: 'sessions-cache.json', + defaultTtl: MS['30_MINUTES'], + saveInterval: 1000, // 1 second + cleanupInterval: MS['5_MINUTES'], + }); + await sessionCache.waitForReady(); + + configCache = new CacheService({ + backend: 'memory', + defaultTtl: MS['30_DAYS'], + cleanupInterval: MS['5_MINUTES'], + maxSize: 100, + }); + + oauthStore = new CacheService({ + backend: 'file', + dataDir: 'data', + fileName: 'oauth-store.json', + saveInterval: 1000, // 1 second + cleanupInterval: MS['10_MINUTES'], + }); + await oauthStore.waitForReady(); + + mcpServersCache = new CacheService({ + backend: 'file', + dataDir: 'data', + fileName: 'mcp-servers-auth.json', + saveInterval: 1000, // 5 seconds + cleanupInterval: MS['5_MINUTES'], + }); + await mcpServersCache.waitForReady(); +} + +export function createCacheBackendsRedis(redisUrl: string): void { + logger.info('Creating cache backends with Redis', redisUrl); + let commonOptions: CacheConfig = { + backend: 'redis', + redisUrl: redisUrl, + defaultTtl: MS['5_MINUTES'], + cleanupInterval: MS['5_MINUTES'], + maxSize: 1000, + }; + + defaultCache = new CacheService({ + ...commonOptions, + dbName: 'default', + }); + + tokenCache = new CacheService({ + backend: 'memory', + defaultTtl: MS['1_MINUTE'], + cleanupInterval: MS['1_MINUTE'], + maxSize: 1000, + }); + + sessionCache = new CacheService({ + ...commonOptions, + dbName: 'session', + }); + + configCache = new CacheService({ + ...commonOptions, + dbName: 'config', + defaultTtl: undefined, + }); + + oauthStore = new CacheService({ + ...commonOptions, + dbName: 'oauth', + defaultTtl: undefined, + }); + + mcpServersCache = new CacheService({ + ...commonOptions, + dbName: 'mcp', + defaultTtl: undefined, + }); +} + +export function createCacheBackendsCF(env: any): void { + let commonOptions: CacheConfig = { + backend: 'cloudflareKV', + env: env, + kvBindingName: 'KV_STORE', + defaultTtl: MS['5_MINUTES'], + }; + defaultCache = new CacheService({ + ...commonOptions, + dbName: 'default', + }); + + tokenCache = new CacheService({ + ...commonOptions, + dbName: 'token', + defaultTtl: MS['10_MINUTES'], + }); + + sessionCache = new CacheService({ + ...commonOptions, + dbName: 'session', + }); + + configCache = new CacheService({ + ...commonOptions, + dbName: 'config', + defaultTtl: MS['30_DAYS'], + }); + + oauthStore = new CacheService({ + ...commonOptions, + dbName: 'oauth', + defaultTtl: undefined, + }); + + mcpServersCache = new CacheService({ + ...commonOptions, + dbName: 'mcp', + defaultTtl: undefined, + }); + + apiRateLimiterCache = new CacheService({ + ...commonOptions, + kvBindingName: 'API_RATE_LIMITER', + dbName: 'api-rate-limiter', + defaultTtl: undefined, + }); +} + +// Re-export types for convenience +export * from './types'; diff --git a/src/shared/services/cache/types.ts b/src/shared/services/cache/types.ts new file mode 100644 index 0000000..8875572 --- /dev/null +++ b/src/shared/services/cache/types.ts @@ -0,0 +1,57 @@ +/** + * @file src/services/cache/types.ts + * Type definitions for the unified cache system + */ + +export interface CacheEntry { + value: T; + expiresAt?: number; + createdAt: number; + metadata?: Record; +} + +export interface CacheOptions { + ttl?: number; // Time to live in milliseconds + namespace?: string; // Cache namespace for organization + metadata?: Record; // Additional metadata +} + +export interface CacheStats { + hits: number; + misses: number; + sets: number; + deletes: number; + size: number; + expired: number; +} + +export interface CacheBackend { + get(key: string, namespace?: string): Promise | null>; + set(key: string, value: T, options?: CacheOptions): Promise; + delete(key: string, namespace?: string): Promise; + clear(namespace?: string): Promise; + has(key: string, namespace?: string): Promise; + keys(namespace?: string): Promise; + getStats(namespace?: string): Promise; + cleanup(): Promise; // Remove expired entries + close(): Promise; // Cleanup resources +} + +export interface CacheConfig { + backend: 'memory' | 'file' | 'redis' | 'cloudflareKV'; + defaultTtl?: number; // Default TTL in milliseconds + cleanupInterval?: number; // Cleanup interval in milliseconds + // File backend options + dataDir?: string; + fileName?: string; + saveInterval?: number; // Debounce save interval + // Redis backend options + redisUrl?: string; + redisOptions?: any; + // Memory backend options + maxSize?: number; // Maximum number of entries + // Cloudflare KV backend options + env?: any; + kvBindingName?: string; + dbName?: string; +} diff --git a/src/shared/services/cache/utils/rateLimiter.ts b/src/shared/services/cache/utils/rateLimiter.ts new file mode 100644 index 0000000..2b478f0 --- /dev/null +++ b/src/shared/services/cache/utils/rateLimiter.ts @@ -0,0 +1,188 @@ +import { Redis, Cluster } from 'ioredis'; +import { RateLimiterKeyTypes } from '../../../../globals'; +import { RedisCacheBackend } from '../backends/redis'; + +const RATE_LIMIT_LUA = ` +local tokensKey = KEYS[1] +local refillKey = KEYS[2] + +local capacity = tonumber(ARGV[1]) +local windowSize = tonumber(ARGV[2]) +local units = tonumber(ARGV[3]) +local now = tonumber(ARGV[4]) +local ttl = tonumber(ARGV[5]) +local consume = tonumber(ARGV[6]) -- 1 = consume, 0 = check only + +-- Reject invalid input +if units <= 0 or capacity <= 0 or windowSize <= 0 then + return {0, -1, -1} +end + +local lastRefill = tonumber(redis.call("GET", refillKey) or "0") +local tokens = tonumber(redis.call("GET", tokensKey) or "-1") + +local tokensModified = false +local refillModified = false + +-- Initialization +if tokens == -1 then + tokens = capacity + tokensModified = true +end + +if lastRefill == 0 then + lastRefill = now + refillModified = true +end + +-- Refill logic +local elapsed = now - lastRefill +if elapsed > 0 then + local rate = capacity / windowSize + local tokensToAdd = math.floor(elapsed * rate) + if tokensToAdd > 0 then + tokens = math.min(tokens + tokensToAdd, capacity) + lastRefill = now -- simpler and avoids drift + tokensModified = true + refillModified = true + end +end + +-- Consume logic +local allowed = 0 +local waitTime = 0 +local currentTokens = tokens + +if tokens >= units then + allowed = 1 + if consume == 1 then + tokens = tokens - units + tokensModified = true + end +else + if tokens > 0 then + tokensModified = true + end + tokens = 0 + local needed = units - currentTokens + local rate = capacity / windowSize + waitTime = (rate > 0) and math.floor(needed / rate) or -1 +end + +-- Save changes +if tokensModified then + redis.call("SET", tokensKey, tokens, "PX", ttl) +end + +if refillModified then + redis.call("SET", refillKey, lastRefill, "PX", ttl) +end + +return {allowed, waitTime, currentTokens} +`; + +class RedisRateLimiter { + private redis: RedisCacheBackend; + private capacity: number; + private windowSize: number; + private tokensKey: string; + private lastRefillKey: string; + private keyTTL: number; + private scriptSha: string | null = null; // To store the SHA1 hash of the script + private keyType: RateLimiterKeyTypes; + private key: string; + + constructor( + redisClient: RedisCacheBackend, + capacity: number, + windowSize: number, + key: string, + keyType: RateLimiterKeyTypes, + ttlFactor: number = 3 // multiplier for TTL + ) { + this.redis = redisClient; + const tag = `{rate:${key}}`; // ensures same hash slot + this.capacity = capacity; + this.windowSize = windowSize; + this.tokensKey = `default:default:${tag}:tokens`; + this.lastRefillKey = `default:default:${tag}:lastRefill`; + this.keyTTL = windowSize * ttlFactor; // dynamic TTL + this.keyType = keyType; + this.key = key; + } + + // Helper to load script if not already loaded and return SHA + private async loadOrGetScriptSha(): Promise { + if (this.scriptSha) { + return this.scriptSha; + } + // Load the script into Redis and get its SHA1 hash + const shaString: any = await this.redis.script('LOAD', RATE_LIMIT_LUA); + this.scriptSha = shaString; + return shaString; + } + + private async executeScript(keys: string[], args: string[]): Promise { + // Get SHA (loads script if not already loaded on current client) + const sha = await this.loadOrGetScriptSha(); + + try { + return await this.redis.evalsha(sha, keys, args); + } catch (error: any) { + if (error.message.includes('NOSCRIPT')) { + // Script not loaded on target node - load it and retry with same SHA + await this.redis.script('LOAD', RATE_LIMIT_LUA); + return await this.redis.evalsha(sha, keys, args); + } + throw error; + } + } + + async checkRateLimit( + units: number, + consumeTokens: boolean = true // Default to true to consume tokens + ): Promise<{ + keyType: RateLimiterKeyTypes; + key: string; + allowed: boolean; + waitTime: number; + currentTokens: number; + }> { + const now = Date.now(); + // Get the SHA, loading the script into Redis if this is the first time + const resp: any = await this.executeScript( + [this.tokensKey, this.lastRefillKey], + [ + this.capacity.toString(), + this.windowSize.toString(), + units.toString(), + now.toString(), + this.keyTTL.toString(), + consumeTokens ? '1' : '0', // Pass consume flag to Lua script + ] + ); + const [allowed, waitTime, currentTokens] = resp; + return { + keyType: this.keyType, + key: this.key, + allowed: allowed === 1, + waitTime: Number(waitTime), + currentTokens: Number(currentTokens), // Return current tokens + }; + } + + async getToken(): Promise { + const cacheEntry = await this.redis.get(this.tokensKey); + return cacheEntry ? cacheEntry.value : null; + } + + async decrementToken( + units: number + ): Promise<{ allowed: boolean; waitTime: number }> { + // Call checkRateLimit ensuring tokens are consumed + const { allowed, waitTime } = await this.checkRateLimit(units, true); + return { allowed, waitTime }; + } +} + +export default RedisRateLimiter; diff --git a/src/shared/utils/logger.ts b/src/shared/utils/logger.ts new file mode 100644 index 0000000..3ad80ee --- /dev/null +++ b/src/shared/utils/logger.ts @@ -0,0 +1,128 @@ +/** + * @file src/utils/logger.ts + * Configurable logger utility for MCP Gateway + */ + +export enum LogLevel { + ERROR = 0, + CRITICAL = 1, // New level for critical information + WARN = 2, + INFO = 3, + DEBUG = 4, +} + +export interface LoggerConfig { + level: LogLevel; + prefix?: string; + timestamp?: boolean; + colors?: boolean; +} + +class Logger { + private config: LoggerConfig; + private colors = { + error: '\x1b[31m', // red + critical: '\x1b[35m', // magenta + warn: '\x1b[33m', // yellow + info: '\x1b[36m', // cyan + debug: '\x1b[37m', // white + reset: '\x1b[0m', + }; + + constructor(config: LoggerConfig) { + this.config = { + timestamp: true, + colors: true, + ...config, + }; + } + + private formatMessage(level: string, message: string): string { + const parts: string[] = []; + + if (this.config.timestamp) { + parts.push(`[${new Date().toISOString()}]`); + } + + if (this.config.prefix) { + parts.push(`[${this.config.prefix}]`); + } + + parts.push(`[${level.toUpperCase()}]`); + parts.push(message); + + return parts.join(' '); + } + + private log(level: LogLevel, levelName: string, message: string, data?: any) { + if (level > this.config.level) return; + + const formattedMessage = this.formatMessage(levelName, message); + const color = this.config.colors + ? this.colors[levelName as keyof typeof this.colors] + : ''; + const reset = this.config.colors ? this.colors.reset : ''; + + if (data !== undefined) { + console.log(`${color}${formattedMessage}${reset}`, data); + } else { + console.log(`${color}${formattedMessage}${reset}`); + } + } + + error(message: string, error?: Error | any) { + if (error instanceof Error) { + this.log(LogLevel.ERROR, 'error', `${message}: ${error.message}`); + if (this.config.level >= LogLevel.DEBUG) { + console.error(error.stack); + } + } else if (error) { + this.log(LogLevel.ERROR, 'error', message, error); + } else { + this.log(LogLevel.ERROR, 'error', message); + } + } + + critical(message: string, data?: any) { + this.log(LogLevel.CRITICAL, 'critical', message, data); + } + + warn(message: string, data?: any) { + this.log(LogLevel.WARN, 'warn', message, data); + } + + info(message: string, data?: any) { + this.log(LogLevel.INFO, 'info', message, data); + } + + debug(message: string, data?: any) { + this.log(LogLevel.DEBUG, 'debug', message, data); + } + + createChild(prefix: string): Logger { + return new Logger({ + ...this.config, + prefix: this.config.prefix ? `${this.config.prefix}:${prefix}` : prefix, + }); + } +} + +// Create default logger instance +const defaultConfig: LoggerConfig = { + level: process.env.LOG_LEVEL + ? LogLevel[process.env.LOG_LEVEL.toUpperCase() as keyof typeof LogLevel] || + LogLevel.ERROR + : process.env.NODE_ENV === 'production' + ? LogLevel.ERROR + : LogLevel.INFO, + timestamp: process.env.LOG_TIMESTAMP !== 'false', + colors: + process.env.LOG_COLORS !== 'false' && process.env.NODE_ENV !== 'production', +}; + +export const logger = new Logger(defaultConfig); + +// Helper to create a logger for a specific component +export function createLogger(prefix: string): Logger { + return logger.createChild(prefix); +} diff --git a/src/start-server.ts b/src/start-server.ts new file mode 100644 index 0000000..853ec12 --- /dev/null +++ b/src/start-server.ts @@ -0,0 +1,215 @@ +#!/usr/bin/env node + +import { serve } from '@hono/node-server'; + +import app from './index'; +import { streamSSE } from 'hono/streaming'; +import { Context } from 'hono'; +import { createNodeWebSocket } from '@hono/node-ws'; +import { realTimeHandlerNode } from './handlers/realtimeHandlerNode'; +import { requestValidator } from './middlewares/requestValidator'; +import conf from '../conf.json'; +import { + adminAuthLoginHandler, + adminAuthMiddleware, + adminAuthSessionStatusHandler, +} from './middlewares/adminAuth'; + +// Extract the port number from the command line arguments +const defaultPort = 8787; +const args = process.argv.slice(2); +const portArg = args.find((arg) => arg.startsWith('--port=')); +const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; + +const isHeadless = args.includes('--headless'); +const hasAdminTokenKey = Object.prototype.hasOwnProperty.call( + conf || {}, + 'admin_token' +); + +// Setup static file serving only if not in headless mode +if ( + !isHeadless && + !( + process.env.NODE_ENV === 'production' || + process.env.ENVIRONMENT === 'production' + ) +) { + const setupStaticServing = async () => { + const { join, dirname } = await import('path'); + const { fileURLToPath } = await import('url'); + const { readFileSync } = await import('fs'); + + const scriptDir = dirname(fileURLToPath(import.meta.url)); + + // Serve the index.html content directly for both routes + const indexPath = join(scriptDir, 'public/index.html'); + const indexContent = readFileSync(indexPath, 'utf-8'); + + const serveIndex = (c: Context) => { + if (!hasAdminTokenKey) { + return c.html( + 'Admin token is required to access the local gateway UI. Please set admin_token in conf.json. see github discussion for more details.' + ); + } + return c.html(indexContent); + }; + + // Set up routes + app.post('/public/auth', adminAuthLoginHandler); + app.get('/public/auth/session', adminAuthSessionStatusHandler); + app.get('/public/logs', serveIndex); + app.get('/public/', serveIndex); + + // Redirect `/public` to `/public/` + app.get('/public', (c: Context) => { + return c.redirect('/public/'); + }); + }; + + // Initialize static file serving + await setupStaticServing(); + + /** + * A helper function to enforce a timeout on SSE sends. + * @param fn A function that returns a Promise (e.g. stream.writeSSE()) + * @param timeoutMs The timeout in milliseconds (default: 2000) + */ + async function sendWithTimeout(fn: () => Promise, timeoutMs = 200) { + const timeoutPromise = new Promise((_, reject) => { + const id = setTimeout(() => { + clearTimeout(id); + reject(new Error('Write timeout')); + }, timeoutMs); + }); + + return Promise.race([fn(), timeoutPromise]); + } + + app.get('/log/stream', adminAuthMiddleware, (c: Context) => { + const clientId = Date.now().toString(); + + // Set headers to prevent caching + c.header('Cache-Control', 'no-cache'); + c.header('X-Accel-Buffering', 'no'); + + return streamSSE(c, async (stream) => { + const addLogClient: any = c.get('addLogClient'); + const removeLogClient: any = c.get('removeLogClient'); + + const client = { + sendLog: (message: any) => + sendWithTimeout(() => stream.writeSSE(message)), + }; + // Add this client to the set of log clients + addLogClient(clientId, client); + + // If the client disconnects (closes the tab, etc.), this signal will be aborted + const onAbort = () => { + removeLogClient(clientId); + }; + c.req.raw.signal.addEventListener('abort', onAbort); + + try { + // Send an initial connection event + await sendWithTimeout(() => + stream.writeSSE({ event: 'connected', data: clientId }) + ); + + // Use an interval instead of a while loop + const heartbeatInterval = setInterval(async () => { + if (c.req.raw.signal.aborted) { + clearInterval(heartbeatInterval); + return; + } + + try { + await sendWithTimeout(() => + stream.writeSSE({ event: 'heartbeat', data: 'pulse' }) + ); + } catch (error) { + // console.error(`Heartbeat failed for client ${clientId}:`, error); + clearInterval(heartbeatInterval); + removeLogClient(clientId); + } + }, 10000); + + // Wait for abort signal + await new Promise((resolve) => { + c.req.raw.signal.addEventListener('abort', () => { + clearInterval(heartbeatInterval); + resolve(undefined); + }); + }); + } catch (error) { + // console.error(`Error in log stream for client ${clientId}:`, error); + } finally { + // Remove this client when the connection is closed + removeLogClient(clientId); + c.req.raw.signal.removeEventListener('abort', onAbort); + } + }); + }); +} + +const { injectWebSocket, upgradeWebSocket } = createNodeWebSocket({ app }); + +app.get( + '/v1/realtime', + requestValidator, + upgradeWebSocket(realTimeHandlerNode) +); + +const server = serve({ + fetch: app.fetch, + port: port, +}); + +const url = `http://localhost:${port}`; + +injectWebSocket(server); + +// Loading animation function +async function showLoadingAnimation() { + const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + let i = 0; + + return new Promise((resolve) => { + const interval = setInterval(() => { + process.stdout.write(`\r${frames[i]} Starting AI Gateway...`); + i = (i + 1) % frames.length; + }, 80); + + // Stop after 1 second + setTimeout(() => { + clearInterval(interval); + process.stdout.write('\r'); + resolve(undefined); + }, 1000); + }); +} + +// Clear the console and show animation before main output +console.clear(); +await showLoadingAnimation(); + +// Main server information with minimal spacing +console.log('\x1b[1m%s\x1b[0m', '🚀 Your AI Gateway is running at:'); +console.log(' ' + '\x1b[1;4;32m%s\x1b[0m', `${url}`); + +// Secondary information on single lines +if (!isHeadless) { + console.log('\n\x1b[90m📱 UI:\x1b[0m \x1b[36m%s\x1b[0m', `${url}/public/`); +} +// console.log('\x1b[90m📚 Docs:\x1b[0m \x1b[36m%s\x1b[0m', 'https://portkey.ai/docs'); + +// Single-line ready message +console.log('\n\x1b[32m✨ Ready for connections!\x1b[0m'); + +process.on('uncaughtException', (err) => { + console.error('Unhandled exception', err); +}); + +process.on('unhandledRejection', (err) => { + console.error('Unhandled rejection', err); +}); diff --git a/src/tests/common.test.ts b/src/tests/common.test.ts new file mode 100644 index 0000000..7964e33 --- /dev/null +++ b/src/tests/common.test.ts @@ -0,0 +1,18 @@ +import Providers from '../providers'; +import testVariables from './resources/testVariables'; +import { executeChatCompletionEndpointTests } from './routeSpecificTestFunctions.ts/chatCompletion'; + +for (const provider in testVariables) { + const variables = testVariables[provider]; + const config = Providers[provider]; + + if (!variables.apiKey) { + console.log(`Skipping ${provider} as API key is not provided`); + continue; + } + + if (config.chatComplete) { + describe(`${provider} /chat/completions endpoint tests:`, () => + executeChatCompletionEndpointTests(provider, variables)); + } +} diff --git a/src/tests/resources/constants.ts b/src/tests/resources/constants.ts new file mode 100644 index 0000000..1aaea3d --- /dev/null +++ b/src/tests/resources/constants.ts @@ -0,0 +1,2 @@ +const baseURL = 'http://localhost'; +export const CHAT_COMPLETIONS_ENDPOINT = `${baseURL}/v1/chat/completions`; diff --git a/src/tests/resources/requestTemplates.ts b/src/tests/resources/requestTemplates.ts new file mode 100644 index 0000000..ae3f2c4 --- /dev/null +++ b/src/tests/resources/requestTemplates.ts @@ -0,0 +1,54 @@ +import { Params } from '../../types/requestBody'; + +const CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST: Params = { + model: 'MODEL_PLACE_HOLDER', + max_tokens: 20, + stream: false, + messages: [ + { + role: 'system', + content: 'You are the half-blood prince', + }, + { + role: 'user', + content: [ + { + type: 'text', + text: 'Can you teach me a useful spell?', + }, + ], + }, + ], +}; + +export const getChatCompleteWithMessageContentArraysRequest = ( + model?: string +) => { + return JSON.stringify({ + ...CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST, + model, + }); +}; + +export const CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST: Params = { + model: 'MODEL_PLACEHOLDER', + max_tokens: 20, + stream: false, + messages: [ + { + role: 'system', + content: 'You are the half-blood prince', + }, + { + role: 'user', + content: 'Can you teach me a useful spell?', + }, + ], +}; + +export const getChatCompleteWithMessageStringRequest = (model?: string) => { + return JSON.stringify({ + ...CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST, + model, + }); +}; diff --git a/src/tests/resources/testVariables.ts b/src/tests/resources/testVariables.ts new file mode 100644 index 0000000..2c5047d --- /dev/null +++ b/src/tests/resources/testVariables.ts @@ -0,0 +1,147 @@ +import Providers from '../../providers'; + +export interface TestVariable { + apiKey?: string; + chatCompletions?: { + model: string; + }; +} + +export interface TestVariables { + [key: keyof typeof Providers]: TestVariable; +} + +const testVariables: TestVariables = { + openai: { + apiKey: process.env.OPENAI_API_KEY, + chatCompletions: { model: 'gpt-3.5-turbo' }, + }, + cohere: { + apiKey: process.env.COHERE_API_KEY, + chatCompletions: { model: 'command-r-plus' }, + }, + anthropic: { + apiKey: process.env.ANTHROPIC_API_KEY, + chatCompletions: { + model: 'claude-3-opus-20240229', + }, + }, + 'azure-openai': { + apiKey: process.env.AZURE_OPENAI_API_KEY, + chatCompletions: { model: '' }, + }, + anyscale: { + apiKey: process.env.ANYSCALE_API_KEY, + chatCompletions: { model: 'j2-light' }, + }, + palm: { + apiKey: process.env.PALM_API_KEY, + chatCompletions: { model: '' }, + }, + 'together-ai': { + apiKey: process.env.TOGETHER_AI_API_KEY, + chatCompletions: { model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, + }, + google: { + apiKey: process.env.GOOGLE_API_KEY, + chatCompletions: { model: 'gemini-1.5-flash' }, + }, + 'vertex-ai': { + apiKey: process.env.VERTEX_AI_API_KEY, + chatCompletions: { model: '' }, + }, + 'perplexity-ai': { + apiKey: process.env.PERPLEXITY_AI_API_KEY, + chatCompletions: { model: 'llama-3-sonar-small-32k-online' }, + }, + 'mistral-ai': { + apiKey: process.env.MISTRAL_AI_API_KEY, + chatCompletions: { model: 'open-mistral-nemo' }, + }, + deepinfra: { + apiKey: process.env.DEEPINFRA_API_KEY, + chatCompletions: { model: 'meta-llama/Meta-Llama-3-8B-Instruct' }, + }, + 'stability-ai': { + apiKey: process.env.STABILITY_AI_API_KEY, + chatCompletions: { model: '' }, + }, + nomic: { + apiKey: process.env.NOMIC_API_KEY, + chatCompletions: { model: '' }, + }, + ollama: { + apiKey: process.env.OLLAMA_API_KEY, + chatCompletions: { model: '' }, + }, + ai21: { + apiKey: process.env.AI21_API_KEY, + chatCompletions: { + model: 'j2-ultra', + }, + }, + bedrock: { + apiKey: process.env.BEDROCK_API_KEY, + chatCompletions: { model: '' }, + }, + groq: { + apiKey: process.env.GROQ_API_KEY, + chatCompletions: { model: 'llama3-8b-8192' }, + }, + segmind: { + apiKey: process.env.SEGMIND_API_KEY, + chatCompletions: { model: '' }, + }, + jina: { + apiKey: process.env.JINA_API_KEY, + chatCompletions: { model: '' }, + }, + 'fireworks-ai': { + apiKey: process.env.FIREWORKS_AI_API_KEY, + chatCompletions: { model: '' }, + }, + 'workers-ai': { + apiKey: process.env.WORKERS_AI_API_KEY, + chatCompletions: { model: '' }, + }, + 'reka-ai': { + apiKey: process.env.REKA_AI_API_KEY, + chatCompletions: { model: '' }, + }, + moonshot: { + apiKey: process.env.MOONSHOT_API_KEY, + chatCompletions: { model: '' }, + }, + openrouter: { + apiKey: process.env.OPENROUTER_API_KEY, + chatCompletions: { model: 'meta-llama/llama-3.1-8b-instruct:free' }, + }, + lingyi: { + apiKey: process.env.LINGYI_API_KEY, + chatCompletions: { model: '' }, + }, + zhipu: { + apiKey: process.env.ZHIPU_API_KEY, + chatCompletions: { model: '' }, + }, + 'novita-ai': { + apiKey: process.env.NOVITA_AI_API_KEY, + chatCompletions: { model: '' }, + }, + monsterapi: { + apiKey: process.env.MONSTERAPI_API_KEY, + chatCompletions: { model: 'meta-llama/Meta-Llama-3-8B-Instruct' }, + }, + predibase: { + apiKey: process.env.PREDIBASE_API_KEY, + chatCompletions: { model: '' }, + }, + nscale: { + apiKey: process.env.NSCALE_API_KEY, + chatCompletions: { + model: 'Qwen/Qwen2.5-Coder-3B-Instruct', + }, + }, +}; + +export default testVariables; diff --git a/src/tests/resources/utils.ts b/src/tests/resources/utils.ts new file mode 100644 index 0000000..ee22d56 --- /dev/null +++ b/src/tests/resources/utils.ts @@ -0,0 +1,10 @@ +export const createDefaultHeaders = ( + provider: string, + authorization: string +) => { + return { + 'x-portkey-provider': provider, + Authorization: authorization, + 'Content-Type': 'application/json', + }; +}; diff --git a/src/tests/routeSpecificTestFunctions.ts/chatCompletion.ts b/src/tests/routeSpecificTestFunctions.ts/chatCompletion.ts new file mode 100644 index 0000000..fc1de17 --- /dev/null +++ b/src/tests/routeSpecificTestFunctions.ts/chatCompletion.ts @@ -0,0 +1,40 @@ +import app from '../..'; +import { CHAT_COMPLETIONS_ENDPOINT } from '../resources/constants'; +import { + getChatCompleteWithMessageContentArraysRequest, + getChatCompleteWithMessageStringRequest, +} from '../resources/requestTemplates'; +import { TestVariable } from '../resources/testVariables'; +import { createDefaultHeaders } from '../resources/utils'; + +export const executeChatCompletionEndpointTests: ( + providerName: string, + providerVariables: TestVariable +) => void = (providerName, providerVariables) => { + const model = providerVariables.chatCompletions?.model; + const apiKey = providerVariables.apiKey; + if (!model || !apiKey) { + console.warn( + `Skipping ${providerName} as it does not have chat completions options` + ); + return; + } + + test(`${providerName} /chat/completions test message strings`, async () => { + const res = await fetch(CHAT_COMPLETIONS_ENDPOINT, { + method: 'POST', + headers: createDefaultHeaders(providerName, apiKey), + body: getChatCompleteWithMessageStringRequest(model), + }); + expect(res.status).toEqual(200); + }); + + test(`${providerName} /chat/completions test message content arrays`, async () => { + const res = await fetch(CHAT_COMPLETIONS_ENDPOINT, { + method: 'POST', + headers: createDefaultHeaders(providerName, apiKey), + body: getChatCompleteWithMessageContentArraysRequest(model), + }); + expect(res.status).toEqual(200); + }); +}; diff --git a/src/types/MessagesRequest.ts b/src/types/MessagesRequest.ts new file mode 100644 index 0000000..a560618 --- /dev/null +++ b/src/types/MessagesRequest.ts @@ -0,0 +1,786 @@ +export interface CacheControlEphemeral { + type: 'ephemeral'; +} + +export interface ServerToolUseBlockParam { + id: string; + + input: unknown; + + name: 'web_search'; + + type: 'server_tool_use'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface WebSearchResultBlockParam { + encrypted_content: string; + + title: string; + + type: 'web_search_result'; + + url: string; + + page_age?: string | null; +} + +export interface WebSearchToolRequestError { + error_code: + | 'invalid_tool_input' + | 'unavailable' + | 'max_uses_exceeded' + | 'too_many_requests' + | 'query_too_long'; + + type: 'web_search_tool_result_error'; +} + +export type WebSearchToolResultBlockParamContent = + | Array + | WebSearchToolRequestError; + +export interface WebSearchToolResultBlockParam { + content: WebSearchToolResultBlockParamContent; + + tool_use_id: string; + + type: 'web_search_tool_result'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface CitationCharLocationParam { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_char_index: number; + + start_char_index: number; + + type: 'char_location'; +} + +export interface CitationPageLocationParam { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_page_number: number; + + start_page_number: number; + + type: 'page_location'; +} + +export interface CitationContentBlockLocationParam { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_block_index: number; + + start_block_index: number; + + type: 'content_block_location'; +} + +export interface CitationWebSearchResultLocationParam { + cited_text: string; + + encrypted_index: string; + + title: string | null; + + type: 'web_search_result_location'; + + url: string; +} + +export type TextCitationParam = + | CitationCharLocationParam + | CitationPageLocationParam + | CitationContentBlockLocationParam + | CitationWebSearchResultLocationParam; + +export interface TextBlockParam { + text: string; + + type: 'text'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; + + citations?: Array | null; +} + +export interface Base64ImageSource { + data: string; + + media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; + + type: 'base64'; +} + +export interface URLImageSource { + type: 'url'; + + media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; + + url: string; +} + +export interface FileImageSource { + type: 'file'; + + file_id: string; +} + +export interface ImageBlockParam { + source: Base64ImageSource | URLImageSource | FileImageSource; + + type: 'image'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface ToolUseBlockParam { + id: string; + + input: unknown; + + name: string; + + type: 'tool_use'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface ToolResultBlockParam { + tool_use_id: string; + + type: 'tool_result'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; + + content?: string | Array; + + is_error?: boolean; +} + +export interface Base64PDFSource { + data: string; + + media_type: 'application/pdf'; + + type: 'base64'; +} + +export interface PlainTextSource { + data: string; + + media_type: 'text/plain'; + + type: 'text'; +} + +export interface ContentBlockSource { + content: string | Array; + + type: 'content'; +} + +export type ContentBlockSourceContent = TextBlockParam | ImageBlockParam; + +export interface URLPDFSource { + type: 'url'; + + url: string; + + media_type?: 'application/pdf'; +} + +export interface CitationsConfigParam { + enabled?: boolean; +} + +export interface DocumentBlockParam { + source: Base64PDFSource | PlainTextSource | ContentBlockSource | URLPDFSource; + + type: 'document'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; + + citations?: CitationsConfigParam; + + context?: string | null; + + title?: string | null; +} + +export interface ThinkingBlockParam { + signature: string; + + thinking: string; + + type: 'thinking'; +} + +export interface RedactedThinkingBlockParam { + data: string; + + type: 'redacted_thinking'; +} + +export type BetaCodeExecutionToolResultErrorCode = + | 'invalid_tool_input' + | 'unavailable' + | 'too_many_requests' + | 'execution_time_exceeded'; + +export interface BetaCodeExecutionToolResultErrorParam { + error_code: BetaCodeExecutionToolResultErrorCode; + + type: 'code_execution_tool_result_error'; +} + +export interface BetaCodeExecutionOutputBlockParam { + file_id: string; + + type: 'code_execution_output'; +} + +export interface BetaCodeExecutionResultBlockParam { + content: Array; + + return_code: number; + + stderr: string; + + stdout: string; + + type: 'code_execution_result'; +} + +export type BetaCodeExecutionToolResultBlockParamContent = + | BetaCodeExecutionToolResultErrorParam + | BetaCodeExecutionResultBlockParam; + +export interface BetaCodeExecutionToolResultBlockParam { + content: BetaCodeExecutionToolResultBlockParamContent; + + tool_use_id: string; + + type: 'code_execution_tool_result'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +/** + * Regular text content. + */ +export type ContentBlockParam = + | ServerToolUseBlockParam + | WebSearchToolResultBlockParam + | TextBlockParam + | ImageBlockParam + | ToolUseBlockParam + | ToolResultBlockParam + | DocumentBlockParam + | ThinkingBlockParam + | RedactedThinkingBlockParam + | BetaCodeExecutionToolResultBlockParam; + +export interface MessageParam { + content: string | Array; + + role: 'user' | 'assistant'; +} + +export interface Metadata { + /** + * An external identifier for the user who is associated with the request. + */ + user_id?: string | null; +} + +export interface ThinkingConfigEnabled { + /** + * Determines how many tokens Claude can use for its internal reasoning process. + */ + budget_tokens: number; + + type: 'enabled'; +} + +export interface ThinkingConfigDisabled { + type: 'disabled'; +} + +export type ThinkingConfigParam = + | ThinkingConfigEnabled + | ThinkingConfigDisabled; + +/** + * The model will use any available tools. + */ +export interface ToolChoiceAny { + type: 'any'; + + /** + * Whether to disable parallel tool use. + * + * Defaults to `false`. If set to `true`, the model will output exactly one tool + * use. + */ + disable_parallel_tool_use?: boolean; +} + +/** + * The model will automatically decide whether to use tools. + */ +export interface ToolChoiceAuto { + type: 'auto'; + + /** + * Whether to disable parallel tool use. + * + * Defaults to `false`. If set to `true`, the model will output at most one tool + * use. + */ + disable_parallel_tool_use?: boolean; +} + +/** + * The model will not be allowed to use tools. + */ +export interface ToolChoiceNone { + type: 'none'; +} + +/** + * The model will use the specified tool with `tool_choice.name`. + */ +export interface ToolChoiceTool { + /** + * The name of the tool to use. + */ + name: string; + + type: 'tool'; + + /** + * Whether to disable parallel tool use. + * + * Defaults to `false`. If set to `true`, the model will output exactly one tool + * use. + */ + disable_parallel_tool_use?: boolean; +} + +export interface ToolInputSchema { + type: 'object'; + + properties?: unknown | null; + + required?: Array | null; + + [k: string]: unknown; +} + +export interface Tool { + /** + * [JSON schema](https://json-schema.org/draft/2020-12) for this tool's input. + * + * This defines the shape of the `input` that your tool accepts and that the model + * will produce. + */ + input_schema: ToolInputSchema; + + /** + * Name of the tool. + * + * This is how the tool will be called by the model and in `tool_use` blocks. + */ + name: string; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; + + /** + * Description of what this tool does. + * + * Tool descriptions should be as detailed as possible. The more information that + * the model has about what the tool is and how to use it, the better it will + * perform. You can use natural language descriptions to reinforce important + * aspects of the tool input JSON schema. + */ + description?: string; + + type?: 'custom' | null; + + /** + * When true, this tool is not loaded into context initially. + * Claude discovers it via Tool Search Tool on-demand. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ + defer_loading?: boolean; + + /** + * List of tool types that can call this tool programmatically. + * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ + allowed_callers?: string[]; + + /** + * Example inputs demonstrating how to use this tool. + * Helps Claude understand usage patterns beyond JSON schema. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ + input_examples?: Record[]; +} + +export interface ToolBash20250124 { + /** + * Name of the tool. + * + * This is how the tool will be called by the model and in `tool_use` blocks. + */ + name: 'bash'; + + type: 'bash_20250124'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface ToolTextEditor20250124 { + /** + * Name of the tool. + * + * This is how the tool will be called by the model and in `tool_use` blocks. + */ + name: 'str_replace_editor'; + + type: 'text_editor_20250124'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export interface WebSearchUserLocation { + type: 'approximate'; + + /** + * The city of the user. + */ + city?: string | null; + + /** + * The two letter + * [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) of the + * user. + */ + country?: string | null; + + /** + * The region of the user. + */ + region?: string | null; + + /** + * The [IANA timezone](https://nodatime.org/TimeZones) of the user. + */ + timezone?: string | null; +} + +export interface WebSearchTool20250305 { + /** + * Name of the tool. + * + * This is how the tool will be called by the model and in `tool_use` blocks. + */ + name: 'web_search'; + + type: 'web_search_20250305'; + + /** + * If provided, only these domains will be included in results. Cannot be used + * alongside `blocked_domains`. + */ + allowed_domains?: Array | null; + + /** + * If provided, these domains will never appear in results. Cannot be used + * alongside `allowed_domains`. + */ + blocked_domains?: Array | null; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; + + /** + * Maximum number of times the tool can be used in the API request. + */ + max_uses?: number | null; + + /** + * Parameters for the user's location. Used to provide more relevant search + * results. + */ + user_location?: WebSearchUserLocation | null; +} + +export interface TextEditor20250429 { + /** + * Name of the tool. + * + * This is how the tool will be called by the model and in `tool_use` blocks. + */ + name: 'str_replace_based_edit_tool'; + + type: 'text_editor_20250429'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +/** + * Tool Search Tool with regex-based search. + * Enables Claude to discover tools on-demand instead of loading all upfront. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface ToolSearchToolRegex { + /** + * Name of the tool search tool. + */ + name: string; + + type: 'tool_search_tool_regex_20251119'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +/** + * Tool Search Tool with BM25-based search. + * Enables Claude to discover tools on-demand instead of loading all upfront. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface ToolSearchToolBM25 { + /** + * Name of the tool search tool. + */ + name: string; + + type: 'tool_search_tool_bm25_20251119'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +/** + * Code Execution Tool for Programmatic Tool Calling. + * Allows Claude to invoke tools from within a code execution environment. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface CodeExecutionTool { + /** + * Name of the code execution tool. + */ + name: string; + + type: 'code_execution_20250825'; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +/** + * Configuration for individual tools within an MCP toolset. + */ +export interface MCPToolConfig { + /** + * When true, this tool is not loaded into context initially. + */ + defer_loading?: boolean; + + /** + * List of tool types that can call this tool programmatically. + */ + allowed_callers?: string[]; +} + +/** + * MCP Toolset for connecting MCP servers. + * Allows deferring loading for entire servers while keeping specific tools loaded. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface MCPToolset { + type: 'mcp_toolset'; + + /** + * Name of the MCP server to connect to. + */ + mcp_server_name: string; + + /** + * Default configuration applied to all tools in this MCP server. + */ + default_config?: MCPToolConfig; + + /** + * Per-tool configuration overrides, keyed by tool name. + */ + configs?: Record; + + /** + * Create a cache control breakpoint at this content block. + */ + cache_control?: CacheControlEphemeral | null; +} + +export type ToolUnion = + | Tool + | ToolBash20250124 + | ToolTextEditor20250124 + | TextEditor20250429 + | WebSearchTool20250305 + | ToolSearchToolRegex + | ToolSearchToolBM25 + | CodeExecutionTool + | MCPToolset; + +/** + * How the model should use the provided tools. The model can use a specific tool, + * any available tool, decide by itself, or not use tools at all. + */ +export type ToolChoice = + | ToolChoiceAuto + | ToolChoiceAny + | ToolChoiceTool + | ToolChoiceNone; + +export interface MessageCreateParamsBase { + /** + * The maximum number of tokens to generate before stopping. + */ + max_tokens: number; + + /** + * Input messages. + */ + messages: Array; + + /** + * The model that will complete your prompt.\n\nSee + */ + model: string; + + /** + * An object describing metadata about the request. + */ + metadata?: Metadata; + + /** + * Determines whether to use priority capacity (if available) or standard capacity + */ + service_tier?: 'auto' | 'standard_only'; + + /** + * Custom text sequences that will cause the model to stop generating. + */ + stop_sequences?: Array; + + /** + * Whether to incrementally stream the response using server-sent events. + */ + stream?: boolean; + + /** + * System prompt. + */ + system?: string | Array; + + /** + * Amount of randomness injected into the response. + */ + temperature?: number; + + /** + * Configuration for enabling Claude's extended thinking. + */ + thinking?: ThinkingConfigParam; + + /** + * How the model should use the provided tools. The model can use a specific tool, + * any available tool, decide by itself, or not use tools at all. + */ + tool_choice?: ToolChoice; + + /** + * Definitions of tools that the model may use. + */ + tools?: Array; + + /** + * Only sample from the top K options for each subsequent token. + */ + top_k?: number; + + /** + * Use nucleus sampling. + */ + top_p?: number; + + // anthropic specific, maybe move this + anthropic_beta?: string; +} diff --git a/src/types/MessagesStreamResponse.ts b/src/types/MessagesStreamResponse.ts new file mode 100644 index 0000000..810b878 --- /dev/null +++ b/src/types/MessagesStreamResponse.ts @@ -0,0 +1,124 @@ +import { + CitationCharLocation, + CitationContentBlockLocation, + CitationPageLocation, + CitationsWebSearchResultLocation, + MessagesResponse, + RedactedThinkingBlock, + ServerToolUseBlock, + ANTHROPIC_STOP_REASON, + TextBlock, + ThinkingBlock, + ToolUseBlock, + Usage, + WebSearchToolResultBlock, +} from './messagesResponse'; + +export interface RawMessageStartEvent { + message: MessagesResponse; + + type: 'message_start'; +} + +export interface RawMessageDelta { + stop_reason: ANTHROPIC_STOP_REASON | null; + + stop_sequence: string | null; +} + +export interface RawMessageDeltaEvent { + delta: RawMessageDelta; + + type: 'message_delta'; + + /** + * Billing and rate-limit usage. + */ + usage: Usage; +} + +export interface RawMessageStopEvent { + type: 'message_stop'; +} + +export interface RawContentBlockStartEvent { + content_block: + | TextBlock + | ToolUseBlock + | ServerToolUseBlock + | WebSearchToolResultBlock + | ThinkingBlock + | RedactedThinkingBlock; + + index: number; + + type: 'content_block_start'; +} + +export interface TextDelta { + text: string; + + type: 'text_delta'; +} + +export interface InputJSONDelta { + partial_json: string; + + type: 'input_json_delta'; +} + +export interface CitationsDelta { + citation: + | CitationCharLocation + | CitationPageLocation + | CitationContentBlockLocation + | CitationsWebSearchResultLocation; + + type: 'citations_delta'; +} + +export interface ThinkingDelta { + thinking: string; + + type: 'thinking_delta'; +} + +export interface SignatureDelta { + signature: string; + + type: 'signature_delta'; +} + +export type RawContentBlockDelta = + | TextDelta + | InputJSONDelta + | CitationsDelta + | ThinkingDelta + | SignatureDelta; + +export interface RawContentBlockDeltaEvent { + delta: RawContentBlockDelta; + + index: number; + + type: 'content_block_delta'; +} + +export interface RawContentBlockStopEvent { + index: number; + + type: 'content_block_stop'; +} + +export interface RawPingEvent { + type: 'ping'; +} + +export type RawMessageStreamEvent = + | RawMessageStartEvent + | RawMessageDeltaEvent + | RawMessageStopEvent + | RawContentBlockStartEvent + | RawContentBlockDeltaEvent + | RawContentBlockStopEvent + | RawPingEvent; diff --git a/src/types/embedRequestBody.ts b/src/types/embedRequestBody.ts new file mode 100644 index 0000000..35bb130 --- /dev/null +++ b/src/types/embedRequestBody.ts @@ -0,0 +1,59 @@ +import { BaseResponse } from '../providers/types'; +import { Options } from './requestBody'; + +type EmbedInput = { + text?: string; + image?: { + url?: string; + base64?: string; + text?: string; // used for image captioning + }; + video?: { + url?: string; + base64?: string; + start_offset?: number; + end_offset?: number; + interval?: number; + text?: string; // used for video captioning + }; +}; + +export interface EmbedParams { + model: string; // The model name to be used as the embedding model + input: string | string[] | EmbedInput[]; // The text or texts to be embedded + user: string; // An identifier for the user making the request + dimensions?: number; // The number of dimensions the resulting output embeddings should have +} + +export interface EmbedRequestBody { + config: { + provider?: string; // The provider of the AI model, e.g., "anthropic", "cohere", "openai" + apiKeyName?: string; // The API key name of the provider + apiKey?: string; // The API key of the provider + mode?: string; + options?: Options[]; + }; + params: EmbedParams; +} + +export interface EmbedResponseData { + object: string; // The type of data object, e.g., "embedding" + embedding?: number[] | number[][]; // The embedding vector(s) + image_embedding?: number[]; + video_embeddings?: { + start_offset: number; + end_offset?: number; + embedding: number[]; + }[]; + index: number; // The index of the data object +} + +export interface EmbedResponse extends BaseResponse { + object: string; // The type of object returned, e.g., "list" + data: EmbedResponseData[]; // The list of data objects + model: string; // The model used to generate the embedding + usage: { + prompt_tokens: number; // The number of tokens in the prompt + total_tokens: number; // The total number of tokens used + }; +} diff --git a/src/types/inputList.ts b/src/types/inputList.ts new file mode 100644 index 0000000..4b1da09 --- /dev/null +++ b/src/types/inputList.ts @@ -0,0 +1,178 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import * as ResponsesAPI from './modelResponses'; + +/** + * A list of Response items. + */ +export interface ResponseItemList { + /** + * A list of items used to generate this response. + */ + data: Array< + | ResponseItemList.Message + | ResponsesAPI.ResponseOutputMessage + | ResponsesAPI.ResponseFileSearchToolCall + | ResponsesAPI.ResponseComputerToolCall + | ResponseItemList.ComputerCallOutput + | ResponsesAPI.ResponseFunctionWebSearch + | ResponsesAPI.ResponseFunctionToolCall + | ResponseItemList.FunctionCallOutput + >; + + /** + * The ID of the first item in the list. + */ + first_id: string; + + /** + * Whether there are more items available. + */ + has_more: boolean; + + /** + * The ID of the last item in the list. + */ + last_id: string; + + /** + * The type of object returned, must be `list`. + */ + object: 'list'; +} + +export namespace ResponseItemList { + export interface Message { + /** + * The unique ID of the message input. + */ + id: string; + + /** + * A list of one or many input items to the model, containing different content + * types. + */ + content: ResponsesAPI.ResponseInputMessageContentList; + + /** + * The role of the message input. One of `user`, `system`, or `developer`. + */ + role: 'user' | 'system' | 'developer'; + + /** + * The status of item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + + /** + * The type of the message input. Always set to `message`. + */ + type?: 'message'; + } + + export interface ComputerCallOutput { + /** + * The unique ID of the computer call tool output. + */ + id: string; + + /** + * The ID of the computer tool call that produced the output. + */ + call_id: string; + + /** + * A computer screenshot image used with the computer use tool. + */ + output: ComputerCallOutput.Output; + + /** + * The type of the computer tool call output. Always `computer_call_output`. + */ + type: 'computer_call_output'; + + /** + * The safety checks reported by the API that have been acknowledged by the + * developer. + */ + acknowledged_safety_checks?: Array; + + /** + * The status of the message input. One of `in_progress`, `completed`, or + * `incomplete`. Populated when input items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + } + + export namespace ComputerCallOutput { + /** + * A computer screenshot image used with the computer use tool. + */ + export interface Output { + /** + * Specifies the event type. For a computer screenshot, this property is always set + * to `computer_screenshot`. + */ + type: 'computer_screenshot'; + + /** + * The identifier of an uploaded file that contains the screenshot. + */ + file_id?: string; + + /** + * The URL of the screenshot image. + */ + image_url?: string; + } + + /** + * A pending safety check for the computer call. + */ + export interface AcknowledgedSafetyCheck { + /** + * The ID of the pending safety check. + */ + id: string; + + /** + * The type of the pending safety check. + */ + code: string; + + /** + * Details about the pending safety check. + */ + message: string; + } + } + + export interface FunctionCallOutput { + /** + * The unique ID of the function call tool output. + */ + id: string; + + /** + * The unique ID of the function tool call generated by the model. + */ + call_id: string; + + /** + * A JSON string of the output of the function tool call. + */ + output: string; + + /** + * The type of the function tool call output. Always `function_call_output`. + */ + type: 'function_call_output'; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + } +} diff --git a/src/types/messagesResponse.ts b/src/types/messagesResponse.ts new file mode 100644 index 0000000..4adc643 --- /dev/null +++ b/src/types/messagesResponse.ts @@ -0,0 +1,354 @@ +export interface CitationCharLocation { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_char_index: number; + + start_char_index: number; + + type: 'char_location'; +} + +export interface CitationPageLocation { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_page_number: number; + + start_page_number: number; + + type: 'page_location'; +} + +export interface CitationContentBlockLocation { + cited_text: string; + + document_index: number; + + document_title: string | null; + + end_block_index: number; + + start_block_index: number; + + type: 'content_block_location'; +} + +export interface CitationsWebSearchResultLocation { + cited_text: string; + + encrypted_index: string; + + title: string | null; + + type: 'web_search_result_location'; + + url: string; +} + +export type TextCitation = + | CitationCharLocation + | CitationPageLocation + | CitationContentBlockLocation + | CitationsWebSearchResultLocation; + +export interface TextBlock { + /** + * Citations supporting the text block. + * + * The type of citation returned will depend on the type of document being cited. + * Citing a PDF results in `page_location`, plain text results in `char_location`, + * and content document results in `content_block_location`. + */ + citations?: Array | null; + + text: string; + + type: 'text'; +} + +/** + * Indicates the tool was called from within a code execution context. + * Present when using Programmatic Tool Calling. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface ToolUseCaller { + /** + * The type of caller (e.g., "code_execution_20250825"). + */ + type: string; + + /** + * The ID of the server tool use block that initiated this call. + */ + tool_id: string; +} + +export interface ToolUseBlock { + id: string; + + input: unknown; + + name: string; + + type: 'tool_use'; + + /** + * Present when this tool was invoked from within a code execution context. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ + caller?: ToolUseCaller; +} + +export interface ServerToolUseBlock { + id: string; + + input: unknown; + + name: 'web_search'; + + type: 'server_tool_use'; +} + +export interface WebSearchToolResultError { + error_code: + | 'invalid_tool_input' + | 'unavailable' + | 'max_uses_exceeded' + | 'too_many_requests' + | 'query_too_long'; + + type: 'web_search_tool_result_error'; +} + +export interface WebSearchResultBlock { + encrypted_content: string; + + page_age: string | null; + + title: string; + + type: 'web_search_result'; + + url: string; +} + +export type WebSearchToolResultBlockContent = + | WebSearchToolResultError + | Array; + +export interface WebSearchToolResultBlock { + content: WebSearchToolResultBlockContent; + + tool_use_id: string; + + type: 'web_search_tool_result'; +} + +/** + * Error codes for code execution tool results. + */ +export type CodeExecutionToolResultErrorCode = + | 'invalid_tool_input' + | 'unavailable' + | 'too_many_requests' + | 'execution_time_exceeded'; + +/** + * Error result from code execution. + */ +export interface CodeExecutionToolResultError { + error_code: CodeExecutionToolResultErrorCode; + + type: 'code_execution_tool_result_error'; +} + +/** + * Output file from code execution. + */ +export interface CodeExecutionOutputBlock { + file_id: string; + + type: 'code_execution_output'; +} + +/** + * Successful result from code execution. + */ +export interface CodeExecutionResultBlock { + content: Array; + + return_code: number; + + stderr: string; + + stdout: string; + + type: 'code_execution_result'; +} + +export type CodeExecutionToolResultBlockContent = + | CodeExecutionToolResultError + | CodeExecutionResultBlock; + +/** + * Result block from Programmatic Tool Calling code execution. + * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). + */ +export interface CodeExecutionToolResultBlock { + content: CodeExecutionToolResultBlockContent; + + tool_use_id: string; + + type: 'code_execution_tool_result'; +} + +export interface ThinkingBlock { + signature: string; + + thinking: string; + + type: 'thinking'; +} + +export interface RedactedThinkingBlock { + data: string; + + type: 'redacted_thinking'; +} + +export type ContentBlock = + | TextBlock + | ToolUseBlock + | ServerToolUseBlock + | WebSearchToolResultBlock + | CodeExecutionToolResultBlock + | ThinkingBlock + | RedactedThinkingBlock; + +export enum ANTHROPIC_STOP_REASON { + end_turn = 'end_turn', + max_tokens = 'max_tokens', + stop_sequence = 'stop_sequence', + tool_use = 'tool_use', + pause_turn = 'pause_turn', + refusal = 'refusal', +} + +export interface ServerToolUsage { + /** + * The number of web search tool requests. + */ + web_search_requests: number; +} + +export interface Usage { + /** + * The number of input tokens used to create the cache entry. + */ + cache_creation_input_tokens?: number | null; + + /** + * The number of input tokens read from the cache. + */ + cache_read_input_tokens?: number | null; + + /** + * The number of input tokens which were used. + */ + input_tokens: number; + + /** + * The number of output tokens which were used. + */ + output_tokens: number; + + /** + * The number of server tool requests. + */ + server_tool_use?: ServerToolUsage | null; + + /** + * If the request used the priority, standard, or batch tier. + */ + service_tier?: 'standard' | 'priority' | 'batch' | null; +} + +export interface MessagesResponse { + /** + * Unique object identifier. + */ + id: string; + + /** + * Content generated by the model. + */ + content: Array; + + /** + * The model that will complete your prompt. + */ + model: string; + + /** + * Conversational role of the generated message. + * + * This will always be `"assistant"`. + */ + role: 'assistant'; + + /** + * The reason that we stopped. + * + * This may be one the following values: + * + * - `"end_turn"`: the model reached a natural stopping point + * - `"max_tokens"`: we exceeded the requested `max_tokens` or the model's maximum + * - `"stop_sequence"`: one of your provided custom `stop_sequences` was generated + * - `"tool_use"`: the model invoked one or more tools + * + * In non-streaming mode this value is always non-null. In streaming mode, it is + * null in the `message_start` event and non-null otherwise. + */ + stop_reason: ANTHROPIC_STOP_REASON | null; + + /** + * Which custom stop sequence was generated, if any. + * + * This value will be a non-null string if one of your custom stop sequences was + * generated. + */ + stop_sequence?: string | null; + + /** + * Object type. + * + * For Messages, this is always `"message"`. + */ + type: 'message'; + + /** + * Billing and rate-limit usage. + * + * Anthropic's API bills and rate-limits by token counts, as tokens represent the + * underlying cost to our systems. + * + * Under the hood, the API transforms requests into a format suitable for the + * model. The model's output then goes through a parsing stage before becoming an + * API response. As a result, the token counts in `usage` will not match one-to-one + * with the exact visible content of an API request or response. + * + * For example, `output_tokens` will be non-zero, even for an empty string response + * from Claude. + * + * Total input tokens in a request is the summation of `input_tokens`, + * `cache_creation_input_tokens`, and `cache_read_input_tokens`. + */ + usage: Usage; +} diff --git a/src/types/modelResponses.ts b/src/types/modelResponses.ts new file mode 100644 index 0000000..39334d9 --- /dev/null +++ b/src/types/modelResponses.ts @@ -0,0 +1,2634 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import * as Shared from './shared'; + +export interface ParsedResponseOutputText extends ResponseOutputText { + parsed: ParsedT | null; +} + +export type ParsedContent = + | ParsedResponseOutputText + | ResponseOutputRefusal; + +export interface ParsedResponseOutputMessage + extends ResponseOutputMessage { + content: ParsedContent[]; +} + +export interface ParsedResponseFunctionToolCall + extends ResponseFunctionToolCall { + parsed_arguments: any; +} + +export type ParsedResponseOutputItem = + | ParsedResponseOutputMessage + | ParsedResponseFunctionToolCall + | ResponseFileSearchToolCall + | ResponseFunctionWebSearch + | ResponseComputerToolCall + | ResponseOutputItemReasoning; + +export interface ParsedResponse extends OpenAIResponse { + output: Array>; + + output_parsed: ParsedT | null; +} + +export type ResponseParseParams = ResponseCreateParamsNonStreaming; + +/** + * A tool that controls a virtual computer. Learn more about the + * [computer tool](https://platform.openai.com/docs/guides/tools-computer-use). + */ +export interface ComputerTool { + /** + * The height of the computer display. + */ + display_height: number; + + /** + * The width of the computer display. + */ + display_width: number; + + /** + * The type of computer environment to control. + */ + environment: 'mac' | 'windows' | 'ubuntu' | 'browser'; + + /** + * The type of the computer use tool. Always `computer_use_preview`. + */ + type: 'computer-preview'; +} + +/** + * A message input to the model with a role indicating instruction following + * hierarchy. Instructions given with the `developer` or `system` role take + * precedence over instructions given with the `user` role. Messages with the + * `assistant` role are presumed to have been generated by the model in previous + * interactions. + */ +export interface EasyInputMessage { + /** + * Text, image, or audio input to the model, used to generate a response. Can also + * contain previous assistant responses. + */ + content: string | ResponseInputMessageContentList; + + /** + * The role of the message input. One of `user`, `assistant`, `system`, or + * `developer`. + */ + role: 'user' | 'assistant' | 'system' | 'developer'; + + /** + * The type of the message input. Always `message`. + */ + type?: 'message'; +} + +/** + * A tool that searches for relevant content from uploaded files. Learn more about + * the + * [file search tool](https://platform.openai.com/docs/guides/tools-file-search). + */ +export interface FileSearchTool { + /** + * The type of the file search tool. Always `file_search`. + */ + type: 'file_search'; + + /** + * The IDs of the vector stores to search. + */ + vector_store_ids: Array; + + /** + * A filter to apply based on file attributes. + */ + filters?: Shared.ComparisonFilter | Shared.CompoundFilter; + + /** + * The maximum number of results to return. This number should be between 1 and 50 + * inclusive. + */ + max_num_results?: number; + + /** + * Ranking options for search. + */ + ranking_options?: FileSearchToolRankingOptions; +} + +export interface FileSearchToolRankingOptions { + /** + * The ranker to use for the file search. + */ + ranker?: 'auto' | 'default-2024-11-15'; + + /** + * The score threshold for the file search, a number between 0 and 1. Numbers + * closer to 1 will attempt to return only the most relevant results, but may + * return fewer results. + */ + score_threshold?: number; +} + +/** + * Defines a function in your own code the model can choose to call. Learn more + * about + * [function calling](https://platform.openai.com/docs/guides/function-calling). + */ +export interface FunctionTool { + /** + * The name of the function to call. + */ + name: string; + + /** + * A JSON schema object describing the parameters of the function. + */ + parameters: Record; + + /** + * Whether to enforce strict parameter validation. Default `true`. + */ + strict: boolean; + + /** + * The type of the function tool. Always `function`. + */ + type: 'function'; + + /** + * A description of the function. Used by the model to determine whether or not to + * call the function. + */ + description?: string | null; +} + +export interface OpenAIResponse { + /** + * Unique identifier for this Response. + */ + id: string; + + /** + * Unix timestamp (in seconds) of when this Response was created. + */ + created_at: number; + + /** + * An error object returned when the model fails to generate a Response. + */ + error: ResponseError | null; + + /** + * Details about why the response is incomplete. + */ + incomplete_details: IncompleteDetails | null; + + /** + * Inserts a system (or developer) message as the first item in the model's + * context. + * + * When using along with `previous_response_id`, the instructions from a previous + * response will be not be carried over to the next response. This makes it simple + * to swap out system (or developer) messages in new responses. + */ + instructions: string | null; + + /** + * Set of 16 key-value pairs that can be attached to an object. This can be useful + * for storing additional information about the object in a structured format, and + * querying for objects via API or the dashboard. + * + * Keys are strings with a maximum length of 64 characters. Values are strings with + * a maximum length of 512 characters. + */ + metadata: Shared.Metadata | null; + + /** + * Model ID used to generate the response, like `gpt-4o` or `o1`. OpenAI offers a + * wide range of models with different capabilities, performance characteristics, + * and price points. Refer to the + * [model guide](https://platform.openai.com/docs/models) to browse and compare + * available models. + */ + model: string; + + /** + * The object type of this resource - always set to `response`. + */ + object: 'response'; + + /** + * An array of content items generated by the model. + * + * - The length and order of items in the `output` array is dependent on the + * model's response. + * - Rather than accessing the first item in the `output` array and assuming it's + * an `assistant` message with the content generated by the model, you might + * consider using the `output_text` property where supported in SDKs. + */ + output: Array; + + /** + * Whether to allow the model to run tool calls in parallel. + */ + parallel_tool_calls: boolean; + + /** + * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + * make the output more random, while lower values like 0.2 will make it more + * focused and deterministic. We generally recommend altering this or `top_p` but + * not both. + */ + temperature: number | null; + + /** + * How the model should select which tool (or tools) to use when generating a + * response. See the `tools` parameter to see how to specify which tools the model + * can call. + */ + tool_choice: ToolChoiceOptions | ToolChoiceTypes | ToolChoiceFunction; + + /** + * An array of tools the model may call while generating a response. You can + * specify which tool to use by setting the `tool_choice` parameter. + * + * The two categories of tools you can provide the model are: + * + * - **Built-in tools**: Tools that are provided by OpenAI that extend the model's + * capabilities, like + * [web search](https://platform.openai.com/docs/guides/tools-web-search) or + * [file search](https://platform.openai.com/docs/guides/tools-file-search). + * Learn more about + * [built-in tools](https://platform.openai.com/docs/guides/tools). + * - **Function calls (custom tools)**: Functions that are defined by you, enabling + * the model to call your own code. Learn more about + * [function calling](https://platform.openai.com/docs/guides/function-calling). + */ + tools: Array; + + /** + * An alternative to sampling with temperature, called nucleus sampling, where the + * model considers the results of the tokens with top_p probability mass. So 0.1 + * means only the tokens comprising the top 10% probability mass are considered. + * + * We generally recommend altering this or `temperature` but not both. + */ + top_p: number | null; + + /** + * An upper bound for the number of tokens that can be generated for a response, + * including visible output tokens and + * [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + */ + max_output_tokens?: number | null; + + /** + * The unique ID of the previous response to the model. Use this to create + * multi-turn conversations. Learn more about + * [conversation state](https://platform.openai.com/docs/guides/conversation-state). + */ + previous_response_id?: string | null; + + /** + * **o-series models only** + * + * Configuration options for + * [reasoning models](https://platform.openai.com/docs/guides/reasoning). + */ + reasoning?: Shared.Reasoning | null; + + /** + * The status of the response generation. One of `completed`, `failed`, + * `in_progress`, or `incomplete`. + */ + status?: ResponseStatus; + + /** + * Configuration options for a text response from the model. Can be plain text or + * structured JSON data. Learn more: + * + * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) + * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) + */ + text?: ResponseTextConfig; + + /** + * The truncation strategy to use for the model response. + * + * - `auto`: If the context of this response and previous ones exceeds the model's + * context window size, the model will truncate the response to fit the context + * window by dropping input items in the middle of the conversation. + * - `disabled` (default): If a model response will exceed the context window size + * for a model, the request will fail with a 400 error. + */ + truncation?: 'auto' | 'disabled' | null; + + /** + * Represents token usage details including input tokens, output tokens, a + * breakdown of output tokens, and the total tokens used. + */ + usage?: ResponseUsage | null; + + /** + * A unique identifier representing your end-user, which can help OpenAI to monitor + * and detect abuse. + * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + */ + user?: string | null; + + store?: boolean; +} + +export interface IncompleteDetails { + /** + * The reason why the response is incomplete. + */ + reason?: 'max_output_tokens' | 'content_filter'; +} + +/** + * Emitted when there is a partial audio response. + */ +export interface ResponseAudioDeltaEvent { + /** + * A chunk of Base64 encoded response audio bytes. + */ + delta: string; + + /** + * The type of the event. Always `response.audio.delta`. + */ + type: 'response.audio.delta'; +} + +/** + * Emitted when the audio response is complete. + */ +export interface ResponseAudioDoneEvent { + /** + * The type of the event. Always `response.audio.done`. + */ + type: 'response.audio.done'; +} + +/** + * Emitted when there is a partial transcript of audio. + */ +export interface ResponseAudioTranscriptDeltaEvent { + /** + * The partial transcript of the audio response. + */ + delta: string; + + /** + * The type of the event. Always `response.audio.transcript.delta`. + */ + type: 'response.audio.transcript.delta'; +} + +/** + * Emitted when the full audio transcript is completed. + */ +export interface ResponseAudioTranscriptDoneEvent { + /** + * The type of the event. Always `response.audio.transcript.done`. + */ + type: 'response.audio.transcript.done'; +} + +/** + * Emitted when a partial code snippet is added by the code interpreter. + */ +export interface ResponseCodeInterpreterCallCodeDeltaEvent { + /** + * The partial code snippet added by the code interpreter. + */ + delta: string; + + /** + * The index of the output item that the code interpreter call is in progress. + */ + output_index: number; + + /** + * The type of the event. Always `response.code_interpreter_call.code.delta`. + */ + type: 'response.code_interpreter_call.code.delta'; +} + +/** + * Emitted when code snippet output is finalized by the code interpreter. + */ +export interface ResponseCodeInterpreterCallCodeDoneEvent { + /** + * The final code snippet output by the code interpreter. + */ + code: string; + + /** + * The index of the output item that the code interpreter call is in progress. + */ + output_index: number; + + /** + * The type of the event. Always `response.code_interpreter_call.code.done`. + */ + type: 'response.code_interpreter_call.code.done'; +} + +/** + * Emitted when the code interpreter call is completed. + */ +export interface ResponseCodeInterpreterCallCompletedEvent { + /** + * A tool call to run code. + */ + code_interpreter_call: ResponseCodeInterpreterToolCall; + + /** + * The index of the output item that the code interpreter call is in progress. + */ + output_index: number; + + /** + * The type of the event. Always `response.code_interpreter_call.completed`. + */ + type: 'response.code_interpreter_call.completed'; +} + +/** + * Emitted when a code interpreter call is in progress. + */ +export interface ResponseCodeInterpreterCallInProgressEvent { + /** + * A tool call to run code. + */ + code_interpreter_call: ResponseCodeInterpreterToolCall; + + /** + * The index of the output item that the code interpreter call is in progress. + */ + output_index: number; + + /** + * The type of the event. Always `response.code_interpreter_call.in_progress`. + */ + type: 'response.code_interpreter_call.in_progress'; +} + +/** + * Emitted when the code interpreter is actively interpreting the code snippet. + */ +export interface ResponseCodeInterpreterCallInterpretingEvent { + /** + * A tool call to run code. + */ + code_interpreter_call: ResponseCodeInterpreterToolCall; + + /** + * The index of the output item that the code interpreter call is in progress. + */ + output_index: number; + + /** + * The type of the event. Always `response.code_interpreter_call.interpreting`. + */ + type: 'response.code_interpreter_call.interpreting'; +} + +/** + * A tool call to run code. + */ +export interface ResponseCodeInterpreterToolCall { + /** + * The unique ID of the code interpreter tool call. + */ + id: string; + + /** + * The code to run. + */ + code: string; + + /** + * The results of the code interpreter tool call. + */ + results: Array< + ResponseCodeInterpreterToolCall.Logs | ResponseCodeInterpreterToolCall.Files + >; + + /** + * The status of the code interpreter tool call. + */ + status: 'in_progress' | 'interpreting' | 'completed'; + + /** + * The type of the code interpreter tool call. Always `code_interpreter_call`. + */ + type: 'code_interpreter_call'; +} + +export namespace ResponseCodeInterpreterToolCall { + /** + * The output of a code interpreter tool call that is text. + */ + export interface Logs { + /** + * The logs of the code interpreter tool call. + */ + logs: string; + + /** + * The type of the code interpreter text output. Always `logs`. + */ + type: 'logs'; + } + + /** + * The output of a code interpreter tool call that is a file. + */ + export interface Files { + files: Array; + + /** + * The type of the code interpreter file output. Always `files`. + */ + type: 'files'; + } + + export namespace Files { + export interface File { + /** + * The ID of the file. + */ + file_id: string; + + /** + * The MIME type of the file. + */ + mime_type: string; + } + } +} + +/** + * Emitted when the model response is complete. + */ +export interface ResponseCompletedEvent { + /** + * Properties of the completed response. + */ + response: OpenAIResponse; + + /** + * The type of the event. Always `response.completed`. + */ + type: 'response.completed'; +} + +/** + * A tool call to a computer use tool. See the + * [computer use guide](https://platform.openai.com/docs/guides/tools-computer-use) + * for more information. + */ +export interface ResponseComputerToolCall { + /** + * The unique ID of the computer call. + */ + id: string; + + /** + * A click action. + */ + action: + | ResponseComputerToolCall.Click + | ResponseComputerToolCall.DoubleClick + | ResponseComputerToolCall.Drag + | ResponseComputerToolCall.Keypress + | ResponseComputerToolCall.Move + | ResponseComputerToolCall.Screenshot + | ResponseComputerToolCall.Scroll + | ResponseComputerToolCall.Type + | ResponseComputerToolCall.Wait + | null; + + /** + * An identifier used when responding to the tool call with output. + */ + call_id: string; + + /** + * The pending safety checks for the computer call. + */ + pending_safety_checks: Array; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status: 'in_progress' | 'completed' | 'incomplete'; + + /** + * The type of the computer call. Always `computer_call`. + */ + type: 'computer_call'; +} + +export namespace ResponseComputerToolCall { + /** + * A click action. + */ + export interface Click { + /** + * Indicates which mouse button was pressed during the click. One of `left`, + * `right`, `wheel`, `back`, or `forward`. + */ + button: 'left' | 'right' | 'wheel' | 'back' | 'forward'; + + /** + * Specifies the event type. For a click action, this property is always set to + * `click`. + */ + type: 'click'; + + /** + * The x-coordinate where the click occurred. + */ + x: number; + + /** + * The y-coordinate where the click occurred. + */ + y: number; + } + + /** + * A double click action. + */ + export interface DoubleClick { + /** + * Specifies the event type. For a double click action, this property is always set + * to `double_click`. + */ + type: 'double_click'; + + /** + * The x-coordinate where the double click occurred. + */ + x: number; + + /** + * The y-coordinate where the double click occurred. + */ + y: number; + } + + /** + * A drag action. + */ + export interface Drag { + /** + * An array of coordinates representing the path of the drag action. Coordinates + * will appear as an array of objects, eg + * + * ``` + * [ + * { x: 100, y: 200 }, + * { x: 200, y: 300 } + * ] + * ``` + */ + path: Array; + + /** + * Specifies the event type. For a drag action, this property is always set to + * `drag`. + */ + type: 'drag'; + } + + export namespace Drag { + /** + * A series of x/y coordinate pairs in the drag path. + */ + export interface Path { + /** + * The x-coordinate. + */ + x: number; + + /** + * The y-coordinate. + */ + y: number; + } + } + + /** + * A collection of keypresses the model would like to perform. + */ + export interface Keypress { + /** + * The combination of keys the model is requesting to be pressed. This is an array + * of strings, each representing a key. + */ + keys: Array; + + /** + * Specifies the event type. For a keypress action, this property is always set to + * `keypress`. + */ + type: 'keypress'; + } + + /** + * A mouse move action. + */ + export interface Move { + /** + * Specifies the event type. For a move action, this property is always set to + * `move`. + */ + type: 'move'; + + /** + * The x-coordinate to move to. + */ + x: number; + + /** + * The y-coordinate to move to. + */ + y: number; + } + + /** + * A screenshot action. + */ + export interface Screenshot { + /** + * Specifies the event type. For a screenshot action, this property is always set + * to `screenshot`. + */ + type: 'screenshot'; + } + + /** + * A scroll action. + */ + export interface Scroll { + /** + * The horizontal scroll distance. + */ + scroll_x: number; + + /** + * The vertical scroll distance. + */ + scroll_y: number; + + /** + * Specifies the event type. For a scroll action, this property is always set to + * `scroll`. + */ + type: 'scroll'; + + /** + * The x-coordinate where the scroll occurred. + */ + x: number; + + /** + * The y-coordinate where the scroll occurred. + */ + y: number; + } + + /** + * An action to type in text. + */ + export interface Type { + /** + * The text to type. + */ + text: string; + + /** + * Specifies the event type. For a type action, this property is always set to + * `type`. + */ + type: 'type'; + } + + /** + * A wait action. + */ + export interface Wait { + /** + * Specifies the event type. For a wait action, this property is always set to + * `wait`. + */ + type: 'wait'; + } + + /** + * A pending safety check for the computer call. + */ + export interface PendingSafetyCheck { + /** + * The ID of the pending safety check. + */ + id: string; + + /** + * The type of the pending safety check. + */ + code: string; + + /** + * Details about the pending safety check. + */ + message: string; + } +} + +/** + * Multi-modal input and output contents. + */ +export type ResponseContent = + | ResponseInputText + | ResponseInputImage + | ResponseInputFile + | ResponseOutputText + | ResponseOutputRefusal; + +/** + * Emitted when a new content part is added. + */ +export interface ResponseContentPartAddedEvent { + /** + * The index of the content part that was added. + */ + content_index: number; + + /** + * The ID of the output item that the content part was added to. + */ + item_id: string; + + /** + * The index of the output item that the content part was added to. + */ + output_index: number; + + /** + * The content part that was added. + */ + part: ResponseOutputText | ResponseOutputRefusal; + + /** + * The type of the event. Always `response.content_part.added`. + */ + type: 'response.content_part.added'; +} + +/** + * Emitted when a content part is done. + */ +export interface ResponseContentPartDoneEvent { + /** + * The index of the content part that is done. + */ + content_index: number; + + /** + * The ID of the output item that the content part was added to. + */ + item_id: string; + + /** + * The index of the output item that the content part was added to. + */ + output_index: number; + + /** + * The content part that is done. + */ + part: ResponseOutputText | ResponseOutputRefusal; + + /** + * The type of the event. Always `response.content_part.done`. + */ + type: 'response.content_part.done'; +} + +/** + * An event that is emitted when a response is created. + */ +export interface ResponseCreatedEvent { + /** + * The response that was created. + */ + response: OpenAIResponse; + + /** + * The type of the event. Always `response.created`. + */ + type: 'response.created'; +} + +/** + * An error object returned when the model fails to generate a Response. + */ +export interface ResponseError { + /** + * The error code for the response. + */ + code: string; + + /** + * A human-readable description of the error. + */ + message: string; +} + +/** + * Emitted when an error occurs. + */ +export interface ResponseErrorEvent { + /** + * The error code. + */ + code: string | null; + + /** + * The error message. + */ + message: string; + + /** + * The error parameter. + */ + param: string | null; + + /** + * The type of the event. Always `error`. + */ + type: 'error'; +} + +/** + * An event that is emitted when a response fails. + */ +export interface ResponseFailedEvent { + /** + * The response that failed. + */ + response: OpenAIResponse; + + /** + * The type of the event. Always `response.failed`. + */ + type: 'response.failed'; +} + +/** + * Emitted when a file search call is completed (results found). + */ +export interface ResponseFileSearchCallCompletedEvent { + /** + * The ID of the output item that the file search call is initiated. + */ + item_id: string; + + /** + * The index of the output item that the file search call is initiated. + */ + output_index: number; + + /** + * The type of the event. Always `response.file_search_call.completed`. + */ + type: 'response.file_search_call.completed'; +} + +/** + * Emitted when a file search call is initiated. + */ +export interface ResponseFileSearchCallInProgressEvent { + /** + * The ID of the output item that the file search call is initiated. + */ + item_id: string; + + /** + * The index of the output item that the file search call is initiated. + */ + output_index: number; + + /** + * The type of the event. Always `response.file_search_call.in_progress`. + */ + type: 'response.file_search_call.in_progress'; +} + +/** + * Emitted when a file search is currently searching. + */ +export interface ResponseFileSearchCallSearchingEvent { + /** + * The ID of the output item that the file search call is initiated. + */ + item_id: string; + + /** + * The index of the output item that the file search call is searching. + */ + output_index: number; + + /** + * The type of the event. Always `response.file_search_call.searching`. + */ + type: 'response.file_search_call.searching'; +} + +/** + * The results of a file search tool call. See the + * [file search guide](https://platform.openai.com/docs/guides/tools-file-search) + * for more information. + */ +export interface ResponseFileSearchToolCall { + /** + * The unique ID of the file search tool call. + */ + id: string; + + /** + * The queries used to search for files. + */ + queries: Array; + + /** + * The status of the file search tool call. One of `in_progress`, `searching`, + * `incomplete` or `failed`, + */ + status: 'in_progress' | 'searching' | 'completed' | 'incomplete' | 'failed'; + + /** + * The type of the file search tool call. Always `file_search_call`. + */ + type: 'file_search_call'; + + /** + * The results of the file search tool call. + */ + results?: Array | null; +} + +export namespace ResponseFileSearchToolCall { + export interface Result { + /** + * Set of 16 key-value pairs that can be attached to an object. This can be useful + * for storing additional information about the object in a structured format, and + * querying for objects via API or the dashboard. Keys are strings with a maximum + * length of 64 characters. Values are strings with a maximum length of 512 + * characters, booleans, or numbers. + */ + attributes?: Record | null; + + /** + * The unique ID of the file. + */ + file_id?: string; + + /** + * The name of the file. + */ + filename?: string; + + /** + * The relevance score of the file - a value between 0 and 1. + */ + score?: number; + + /** + * The text that was retrieved from the file. + */ + text?: string; + } +} + +/** + * An object specifying the format that the model must output. + * + * Configuring `{ "type": "json_schema" }` enables Structured Outputs, which + * ensures the model will match your supplied JSON schema. Learn more in the + * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + * + * The default format is `{ "type": "text" }` with no additional options. + * + * **Not recommended for gpt-4o and newer models:** + * + * Setting to `{ "type": "json_object" }` enables the older JSON mode, which + * ensures the message the model generates is valid JSON. Using `json_schema` is + * preferred for models that support it. + */ +export type ResponseFormatTextConfig = + | Shared.ResponseFormatText + | ResponseFormatTextJSONSchemaConfig + | Shared.ResponseFormatJSONObject; + +/** + * JSON Schema response format. Used to generate structured JSON responses. Learn + * more about + * [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs). + */ +export interface ResponseFormatTextJSONSchemaConfig { + /** + * The schema for the response format, described as a JSON Schema object. Learn how + * to build JSON schemas [here](https://json-schema.org/). + */ + schema: Record; + + /** + * The type of response format being defined. Always `json_schema`. + */ + type: 'json_schema'; + + /** + * A description of what the response format is for, used by the model to determine + * how to respond in the format. + */ + description?: string; + + /** + * The name of the response format. Must be a-z, A-Z, 0-9, or contain underscores + * and dashes, with a maximum length of 64. + */ + name?: string; + + /** + * Whether to enable strict schema adherence when generating the output. If set to + * true, the model will always follow the exact schema defined in the `schema` + * field. Only a subset of JSON Schema is supported when `strict` is `true`. To + * learn more, read the + * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + */ + strict?: boolean | null; +} + +/** + * Emitted when there is a partial function-call arguments delta. + */ +export interface ResponseFunctionCallArgumentsDeltaEvent { + /** + * The function-call arguments delta that is added. + */ + delta: string; + + /** + * The ID of the output item that the function-call arguments delta is added to. + */ + item_id: string; + + /** + * The index of the output item that the function-call arguments delta is added to. + */ + output_index: number; + + /** + * The type of the event. Always `response.function_call_arguments.delta`. + */ + type: 'response.function_call_arguments.delta'; +} + +/** + * Emitted when function-call arguments are finalized. + */ +export interface ResponseFunctionCallArgumentsDoneEvent { + /** + * The function-call arguments. + */ + arguments: string; + + /** + * The ID of the item. + */ + item_id: string; + + /** + * The index of the output item. + */ + output_index: number; + + type: 'response.function_call_arguments.done'; +} + +/** + * A tool call to run a function. See the + * [function calling guide](https://platform.openai.com/docs/guides/function-calling) + * for more information. + */ +export interface ResponseFunctionToolCall { + /** + * The unique ID of the function tool call. + */ + id: string; + + /** + * A JSON string of the arguments to pass to the function. + */ + arguments: string; + + /** + * The unique ID of the function tool call generated by the model. + */ + call_id: string; + + /** + * The name of the function to run. + */ + name: string; + + /** + * The type of the function tool call. Always `function_call`. + */ + type: 'function_call'; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; +} + +/** + * The results of a web search tool call. See the + * [web search guide](https://platform.openai.com/docs/guides/tools-web-search) for + * more information. + */ +export interface ResponseFunctionWebSearch { + /** + * The unique ID of the web search tool call. + */ + id: string; + + /** + * The status of the web search tool call. + */ + status: 'in_progress' | 'searching' | 'completed' | 'failed'; + + /** + * The type of the web search tool call. Always `web_search_call`. + */ + type: 'web_search_call'; +} + +/** + * Emitted when the response is in progress. + */ +export interface ResponseInProgressEvent { + /** + * The response that is in progress. + */ + response: OpenAIResponse; + + /** + * The type of the event. Always `response.in_progress`. + */ + type: 'response.in_progress'; +} + +/** + * Specify additional output data to include in the model response. Currently + * supported values are: + * + * - `file_search_call.results`: Include the search results of the file search tool + * call. + * - `message.input_image.image_url`: Include image urls from the input message. + * - `computer_call_output.output.image_url`: Include image urls from the computer + * call output. + */ +export type ResponseIncludable = + | 'file_search_call.results' + | 'message.input_image.image_url' + | 'computer_call_output.output.image_url'; + +/** + * An event that is emitted when a response finishes as incomplete. + */ +export interface ResponseIncompleteEvent { + /** + * The response that was incomplete. + */ + response: OpenAIResponse; + + /** + * The type of the event. Always `response.incomplete`. + */ + type: 'response.incomplete'; +} + +/** + * A list of one or many input items to the model, containing different content + * types. + */ +export type ResponseInput = Array; + +/** + * An audio input to the model. + */ +export interface ResponseInputAudio { + /** + * Base64-encoded audio data. + */ + data: string; + + /** + * The format of the audio data. Currently supported formats are `mp3` and `wav`. + */ + format: 'mp3' | 'wav'; + + /** + * The type of the input item. Always `input_audio`. + */ + type: 'input_audio'; +} + +/** + * A text input to the model. + */ +export type ResponseInputContent = + | ResponseInputText + | ResponseInputImage + | ResponseInputFile; + +/** + * A file input to the model. + */ +export interface ResponseInputFile { + /** + * The type of the input item. Always `input_file`. + */ + type: 'input_file'; + + /** + * The content of the file to be sent to the model. + */ + file_data?: string; + + /** + * The ID of the file to be sent to the model. + */ + file_id?: string; + + /** + * The name of the file to be sent to the model. + */ + filename?: string; +} + +/** + * An image input to the model. Learn about + * [image inputs](https://platform.openai.com/docs/guides/vision). + */ +export interface ResponseInputImage { + /** + * The detail level of the image to be sent to the model. One of `high`, `low`, or + * `auto`. Defaults to `auto`. + */ + detail: 'high' | 'low' | 'auto'; + + /** + * The type of the input item. Always `input_image`. + */ + type: 'input_image'; + + /** + * The ID of the file to be sent to the model. + */ + file_id?: string | null; + + /** + * The URL of the image to be sent to the model. A fully qualified URL or base64 + * encoded image in a data URL. + */ + image_url?: string | null; +} + +/** + * A message input to the model with a role indicating instruction following + * hierarchy. Instructions given with the `developer` or `system` role take + * precedence over instructions given with the `user` role. Messages with the + * `assistant` role are presumed to have been generated by the model in previous + * interactions. + */ +export type ResponseInputItem = + | EasyInputMessage + | ResponseInputItem.Message + | ResponseOutputMessage + | ResponseFileSearchToolCall + | ResponseComputerToolCall + | ResponseInputItem.ComputerCallOutput + | ResponseFunctionWebSearch + | ResponseFunctionToolCall + | ResponseInputItem.FunctionCallOutput + | ResponseInputItem.Reasoning + | ResponseInputItem.ItemReference; + +export namespace ResponseInputItem { + /** + * A message input to the model with a role indicating instruction following + * hierarchy. Instructions given with the `developer` or `system` role take + * precedence over instructions given with the `user` role. + */ + export interface Message { + /** + * A list of one or many input items to the model, containing different content + * types. + */ + content: ResponseInputMessageContentList; + + /** + * The role of the message input. One of `user`, `system`, or `developer`. + */ + role: 'user' | 'system' | 'developer'; + + /** + * The status of item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + + /** + * The type of the message input. Always set to `message`. + */ + type?: 'message'; + } + + /** + * The output of a computer tool call. + */ + export interface ComputerCallOutput { + /** + * The ID of the computer tool call that produced the output. + */ + call_id: string; + + /** + * A computer screenshot image used with the computer use tool. + */ + output: ComputerCallOutput.Output; + + /** + * The type of the computer tool call output. Always `computer_call_output`. + */ + type: 'computer_call_output'; + + /** + * The ID of the computer tool call output. + */ + id?: string; + + /** + * The safety checks reported by the API that have been acknowledged by the + * developer. + */ + acknowledged_safety_checks?: Array; + + /** + * The status of the message input. One of `in_progress`, `completed`, or + * `incomplete`. Populated when input items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + } + + export namespace ComputerCallOutput { + /** + * A computer screenshot image used with the computer use tool. + */ + export interface Output { + /** + * Specifies the event type. For a computer screenshot, this property is always set + * to `computer_screenshot`. + */ + type: 'computer_screenshot'; + + /** + * The identifier of an uploaded file that contains the screenshot. + */ + file_id?: string; + + /** + * The URL of the screenshot image. + */ + image_url?: string; + } + + /** + * A pending safety check for the computer call. + */ + export interface AcknowledgedSafetyCheck { + /** + * The ID of the pending safety check. + */ + id: string; + + /** + * The type of the pending safety check. + */ + code: string; + + /** + * Details about the pending safety check. + */ + message: string; + } + } + + /** + * The output of a function tool call. + */ + export interface FunctionCallOutput { + /** + * The unique ID of the function tool call generated by the model. + */ + call_id: string; + + /** + * A JSON string of the output of the function tool call. + */ + output: string; + + /** + * The type of the function tool call output. Always `function_call_output`. + */ + type: 'function_call_output'; + + /** + * The unique ID of the function tool call output. Populated when this item is + * returned via API. + */ + id?: string; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + } + + /** + * A description of the chain of thought used by a reasoning model while generating + * a response. + */ + export interface Reasoning { + /** + * The unique identifier of the reasoning content. + */ + id: string; + + /** + * Reasoning text contents. + */ + content: Array; + + /** + * The type of the object. Always `reasoning`. + */ + type: 'reasoning'; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + } + + export namespace Reasoning { + export interface Content { + /** + * A short summary of the reasoning used by the model when generating the response. + */ + text: string; + + /** + * The type of the object. Always `text`. + */ + type: 'reasoning_summary'; + } + } + + /** + * An internal identifier for an item to reference. + */ + export interface ItemReference { + /** + * The ID of the item to reference. + */ + id: string; + + /** + * The type of item to reference. Always `item_reference`. + */ + type: 'item_reference'; + } +} + +/** + * A list of one or many input items to the model, containing different content + * types. + */ +export type ResponseInputMessageContentList = Array; + +/** + * A text input to the model. + */ +export interface ResponseInputText { + /** + * The text input to the model. + */ + text: string; + + /** + * The type of the input item. Always `input_text`. + */ + type: 'input_text'; +} + +/** + * An audio output from the model. + */ +export interface ResponseOutputAudio { + /** + * Base64-encoded audio data from the model. + */ + data: string; + + /** + * The transcript of the audio data from the model. + */ + transcript: string; + + /** + * The type of the output audio. Always `output_audio`. + */ + type: 'output_audio'; +} + +/** + * An output message from the model. + */ +export type ResponseOutputItem = + | ResponseOutputMessage + | ResponseFileSearchToolCall + | ResponseFunctionToolCall + | ResponseFunctionWebSearch + | ResponseComputerToolCall + | ResponseOutputItemReasoning; + +export interface ResponseOutputItemReasoning { + /** + * The unique identifier of the reasoning content. + */ + id: string; + + /** + * Reasoning text contents. + */ + content?: { + /** + * A short summary of the reasoning used by the model when generating the response. + */ + text: string; + + /** + * The type of the object. Always `text`. + */ + type: 'reasoning_summary'; + }; + + /** + * The type of the object. Always `reasoning`. + */ + type: 'reasoning'; + + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; + + /** + * A short summary of the reasoning used by the model when generating the response. + */ + summary?: any; +} + +/** + * Emitted when a new output item is added. + */ +export interface ResponseOutputItemAddedEvent { + /** + * The output item that was added. + */ + item: ResponseOutputItem; + + /** + * The index of the output item that was added. + */ + output_index: number; + + /** + * The type of the event. Always `response.output_item.added`. + */ + type: 'response.output_item.added'; +} + +/** + * Emitted when an output item is marked done. + */ +export interface ResponseOutputItemDoneEvent { + /** + * The output item that was marked done. + */ + item: ResponseOutputItem; + + /** + * The index of the output item that was marked done. + */ + output_index: number; + + /** + * The type of the event. Always `response.output_item.done`. + */ + type: 'response.output_item.done'; +} + +/** + * An output message from the model. + */ +export interface ResponseOutputMessage { + /** + * The unique ID of the output message. + */ + id: string; + + /** + * The content of the output message. + */ + content: Array; + + /** + * The role of the output message. Always `assistant`. + */ + role: 'assistant'; + + /** + * The status of the message input. One of `in_progress`, `completed`, or + * `incomplete`. Populated when input items are returned via API. + */ + status: 'in_progress' | 'completed' | 'incomplete'; + + /** + * The type of the output message. Always `message`. + */ + type: 'message'; +} + +/** + * A refusal from the model. + */ +export interface ResponseOutputRefusal { + /** + * The refusal explanationfrom the model. + */ + refusal: string; + + /** + * The type of the refusal. Always `refusal`. + */ + type: 'refusal'; +} + +/** + * A text output from the model. + */ +export interface ResponseOutputText { + /** + * The annotations of the text output. + */ + annotations: Array< + | ResponseOutputText.FileCitation + | ResponseOutputText.URLCitation + | ResponseOutputText.FilePath + >; + + /** + * The text output from the model. + */ + text: string; + + /** + * The type of the output text. Always `output_text`. + */ + type: 'output_text'; +} + +export namespace ResponseOutputText { + /** + * A citation to a file. + */ + export interface FileCitation { + /** + * The ID of the file. + */ + file_id: string; + + /** + * The index of the file in the list of files. + */ + index: number; + + /** + * The type of the file citation. Always `file_citation`. + */ + type: 'file_citation'; + } + + /** + * A citation for a web resource used to generate a model response. + */ + export interface URLCitation { + /** + * The index of the last character of the URL citation in the message. + */ + end_index: number; + + /** + * The index of the first character of the URL citation in the message. + */ + start_index: number; + + /** + * The title of the web resource. + */ + title: string; + + /** + * The type of the URL citation. Always `url_citation`. + */ + type: 'url_citation'; + + /** + * The URL of the web resource. + */ + url: string; + } + + /** + * A path to a file. + */ + export interface FilePath { + /** + * The ID of the file. + */ + file_id: string; + + /** + * The index of the file in the list of files. + */ + index: number; + + /** + * The type of the file path. Always `file_path`. + */ + type: 'file_path'; + } +} + +/** + * Emitted when there is a partial refusal text. + */ +export interface ResponseRefusalDeltaEvent { + /** + * The index of the content part that the refusal text is added to. + */ + content_index: number; + + /** + * The refusal text that is added. + */ + delta: string; + + /** + * The ID of the output item that the refusal text is added to. + */ + item_id: string; + + /** + * The index of the output item that the refusal text is added to. + */ + output_index: number; + + /** + * The type of the event. Always `response.refusal.delta`. + */ + type: 'response.refusal.delta'; +} + +/** + * Emitted when refusal text is finalized. + */ +export interface ResponseRefusalDoneEvent { + /** + * The index of the content part that the refusal text is finalized. + */ + content_index: number; + + /** + * The ID of the output item that the refusal text is finalized. + */ + item_id: string; + + /** + * The index of the output item that the refusal text is finalized. + */ + output_index: number; + + /** + * The refusal text that is finalized. + */ + refusal: string; + + /** + * The type of the event. Always `response.refusal.done`. + */ + type: 'response.refusal.done'; +} + +/** + * The status of the response generation. One of `completed`, `failed`, + * `in_progress`, or `incomplete`. + */ +export type ResponseStatus = + | 'completed' + | 'failed' + | 'in_progress' + | 'incomplete'; + +/** + * Emitted when there is a partial audio response. + */ +export type ResponseStreamEvent = + | ResponseAudioDeltaEvent + | ResponseAudioDoneEvent + | ResponseAudioTranscriptDeltaEvent + | ResponseAudioTranscriptDoneEvent + | ResponseCodeInterpreterCallCodeDeltaEvent + | ResponseCodeInterpreterCallCodeDoneEvent + | ResponseCodeInterpreterCallCompletedEvent + | ResponseCodeInterpreterCallInProgressEvent + | ResponseCodeInterpreterCallInterpretingEvent + | ResponseCompletedEvent + | ResponseContentPartAddedEvent + | ResponseContentPartDoneEvent + | ResponseCreatedEvent + | ResponseErrorEvent + | ResponseFileSearchCallCompletedEvent + | ResponseFileSearchCallInProgressEvent + | ResponseFileSearchCallSearchingEvent + | ResponseFunctionCallArgumentsDeltaEvent + | ResponseFunctionCallArgumentsDoneEvent + | ResponseInProgressEvent + | ResponseFailedEvent + | ResponseIncompleteEvent + | ResponseOutputItemAddedEvent + | ResponseOutputItemDoneEvent + | ResponseRefusalDeltaEvent + | ResponseRefusalDoneEvent + | ResponseTextAnnotationDeltaEvent + | ResponseTextDeltaEvent + | ResponseTextDoneEvent + | ResponseWebSearchCallCompletedEvent + | ResponseWebSearchCallInProgressEvent + | ResponseWebSearchCallSearchingEvent; + +/** + * Emitted when a text annotation is added. + */ +export interface ResponseTextAnnotationDeltaEvent { + /** + * A citation to a file. + */ + annotation: + | ResponseTextAnnotationDeltaEvent.FileCitation + | ResponseTextAnnotationDeltaEvent.URLCitation + | ResponseTextAnnotationDeltaEvent.FilePath; + + /** + * The index of the annotation that was added. + */ + annotation_index: number; + + /** + * The index of the content part that the text annotation was added to. + */ + content_index: number; + + /** + * The ID of the output item that the text annotation was added to. + */ + item_id: string; + + /** + * The index of the output item that the text annotation was added to. + */ + output_index: number; + + /** + * The type of the event. Always `response.output_text.annotation.added`. + */ + type: 'response.output_text.annotation.added'; +} + +export namespace ResponseTextAnnotationDeltaEvent { + /** + * A citation to a file. + */ + export interface FileCitation { + /** + * The ID of the file. + */ + file_id: string; + + /** + * The index of the file in the list of files. + */ + index: number; + + /** + * The type of the file citation. Always `file_citation`. + */ + type: 'file_citation'; + } + + /** + * A citation for a web resource used to generate a model response. + */ + export interface URLCitation { + /** + * The index of the last character of the URL citation in the message. + */ + end_index: number; + + /** + * The index of the first character of the URL citation in the message. + */ + start_index: number; + + /** + * The title of the web resource. + */ + title: string; + + /** + * The type of the URL citation. Always `url_citation`. + */ + type: 'url_citation'; + + /** + * The URL of the web resource. + */ + url: string; + } + + /** + * A path to a file. + */ + export interface FilePath { + /** + * The ID of the file. + */ + file_id: string; + + /** + * The index of the file in the list of files. + */ + index: number; + + /** + * The type of the file path. Always `file_path`. + */ + type: 'file_path'; + } +} + +/** + * Configuration options for a text response from the model. Can be plain text or + * structured JSON data. Learn more: + * + * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) + * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) + */ +export interface ResponseTextConfig { + /** + * An object specifying the format that the model must output. + * + * Configuring `{ "type": "json_schema" }` enables Structured Outputs, which + * ensures the model will match your supplied JSON schema. Learn more in the + * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + * + * The default format is `{ "type": "text" }` with no additional options. + * + * **Not recommended for gpt-4o and newer models:** + * + * Setting to `{ "type": "json_object" }` enables the older JSON mode, which + * ensures the message the model generates is valid JSON. Using `json_schema` is + * preferred for models that support it. + */ + format?: ResponseFormatTextConfig; +} + +/** + * Emitted when there is an additional text delta. + */ +export interface ResponseTextDeltaEvent { + /** + * The index of the content part that the text delta was added to. + */ + content_index: number; + + /** + * The text delta that was added. + */ + delta: string; + + /** + * The ID of the output item that the text delta was added to. + */ + item_id: string; + + /** + * The index of the output item that the text delta was added to. + */ + output_index: number; + + /** + * The type of the event. Always `response.output_text.delta`. + */ + type: 'response.output_text.delta'; +} + +/** + * Emitted when text content is finalized. + */ +export interface ResponseTextDoneEvent { + /** + * The index of the content part that the text content is finalized. + */ + content_index: number; + + /** + * The ID of the output item that the text content is finalized. + */ + item_id: string; + + /** + * The index of the output item that the text content is finalized. + */ + output_index: number; + + /** + * The text content that is finalized. + */ + text: string; + + /** + * The type of the event. Always `response.output_text.done`. + */ + type: 'response.output_text.done'; +} + +/** + * Represents token usage details including input tokens, output tokens, a + * breakdown of output tokens, and the total tokens used. + */ +export interface ResponseUsage { + /** + * The number of input tokens. + */ + input_tokens: number; + + /** + * The number of output tokens. + */ + output_tokens: number; + + /** + * A detailed breakdown of the output tokens. + */ + output_tokens_details: ResponseUsage.OutputTokensDetails; + + /** + * The total number of tokens used. + */ + total_tokens: number; +} + +export namespace ResponseUsage { + /** + * A detailed breakdown of the output tokens. + */ + export interface OutputTokensDetails { + /** + * The number of reasoning tokens. + */ + reasoning_tokens: number; + } +} + +/** + * Emitted when a web search call is completed. + */ +export interface ResponseWebSearchCallCompletedEvent { + /** + * Unique ID for the output item associated with the web search call. + */ + item_id: string; + + /** + * The index of the output item that the web search call is associated with. + */ + output_index: number; + + /** + * The type of the event. Always `response.web_search_call.completed`. + */ + type: 'response.web_search_call.completed'; +} + +/** + * Emitted when a web search call is initiated. + */ +export interface ResponseWebSearchCallInProgressEvent { + /** + * Unique ID for the output item associated with the web search call. + */ + item_id: string; + + /** + * The index of the output item that the web search call is associated with. + */ + output_index: number; + + /** + * The type of the event. Always `response.web_search_call.in_progress`. + */ + type: 'response.web_search_call.in_progress'; +} + +/** + * Emitted when a web search call is executing. + */ +export interface ResponseWebSearchCallSearchingEvent { + /** + * Unique ID for the output item associated with the web search call. + */ + item_id: string; + + /** + * The index of the output item that the web search call is associated with. + */ + output_index: number; + + /** + * The type of the event. Always `response.web_search_call.searching`. + */ + type: 'response.web_search_call.searching'; +} + +/** + * A tool that searches for relevant content from uploaded files. Learn more about + * the + * [file search tool](https://platform.openai.com/docs/guides/tools-file-search). + */ +export type Tool = FileSearchTool | FunctionTool | ComputerTool | WebSearchTool; + +/** + * Use this option to force the model to call a specific function. + */ +export interface ToolChoiceFunction { + /** + * The name of the function to call. + */ + name: string; + + /** + * For function calling, the type is always `function`. + */ + type: 'function'; +} + +/** + * Controls which (if any) tool is called by the model. + * + * `none` means the model will not call any tool and instead generates a message. + * + * `auto` means the model can pick between generating a message or calling one or + * more tools. + * + * `required` means the model must call one or more tools. + */ +export type ToolChoiceOptions = 'none' | 'auto' | 'required'; + +/** + * Indicates that the model should use a built-in tool to generate a response. + * [Learn more about built-in tools](https://platform.openai.com/docs/guides/tools). + */ +export interface ToolChoiceTypes { + /** + * The type of hosted tool the model should to use. Learn more about + * [built-in tools](https://platform.openai.com/docs/guides/tools). + * + * Allowed values are: + * + * - `file_search` + * - `web_search_preview` + * - `computer_use_preview` + */ + type: + | 'file_search' + | 'web_search_preview' + | 'computer_use_preview' + | 'web_search_preview_2025_03_11'; +} + +/** + * This tool searches the web for relevant results to use in a response. Learn more + * about the + * [web search tool](https://platform.openai.com/docs/guides/tools-web-search). + */ +export interface WebSearchTool { + /** + * The type of the web search tool. One of: + * + * - `web_search_preview` + * - `web_search_preview_2025_03_11` + */ + type: 'web_search_preview' | 'web_search_preview_2025_03_11'; + + /** + * High level guidance for the amount of context window space to use for the + * search. One of `low`, `medium`, or `high`. `medium` is the default. + */ + search_context_size?: 'low' | 'medium' | 'high'; + + user_location?: WebSearchToolUserLocation | null; +} + +interface WebSearchToolUserLocation { + /** + * The type of location approximation. Always `approximate`. + */ + type: 'approximate'; + + /** + * Free text input for the city of the user, e.g. `San Francisco`. + */ + city?: string; + + /** + * The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of + * the user, e.g. `US`. + */ + country?: string; + + /** + * Free text input for the region of the user, e.g. `California`. + */ + region?: string; + + /** + * The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the + * user, e.g. `America/Los_Angeles`. + */ + timezone?: string; +} + +export type ResponseCreateParams = + | ResponseCreateParamsNonStreaming + | ResponseCreateParamsStreaming; + +export interface ResponseCreateParamsBase { + /** + * Text, image, or file inputs to the model, used to generate a response. + * + * Learn more: + * + * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) + * - [Image inputs](https://platform.openai.com/docs/guides/images) + * - [File inputs](https://platform.openai.com/docs/guides/pdf-files) + * - [Conversation state](https://platform.openai.com/docs/guides/conversation-state) + * - [Function calling](https://platform.openai.com/docs/guides/function-calling) + */ + input: string | ResponseInput; + + /** + * Model ID used to generate the response, like `gpt-4o` or `o1`. OpenAI offers a + * wide range of models with different capabilities, performance characteristics, + * and price points. Refer to the + * [model guide](https://platform.openai.com/docs/models) to browse and compare + * available models. + */ + model: string; + + /** + * Specify additional output data to include in the model response. Currently + * supported values are: + * + * - `file_search_call.results`: Include the search results of the file search tool + * call. + * - `message.input_image.image_url`: Include image urls from the input message. + * - `computer_call_output.output.image_url`: Include image urls from the computer + * call output. + */ + include?: Array | null; + + /** + * Inserts a system (or developer) message as the first item in the model's + * context. + * + * When using along with `previous_response_id`, the instructions from a previous + * response will be not be carried over to the next response. This makes it simple + * to swap out system (or developer) messages in new responses. + */ + instructions?: string | null; + + /** + * An upper bound for the number of tokens that can be generated for a response, + * including visible output tokens and + * [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + */ + max_output_tokens?: number | null; + + /** + * Set of 16 key-value pairs that can be attached to an object. This can be useful + * for storing additional information about the object in a structured format, and + * querying for objects via API or the dashboard. + * + * Keys are strings with a maximum length of 64 characters. Values are strings with + * a maximum length of 512 characters. + */ + metadata?: Shared.Metadata | null; + + /** + * Whether to allow the model to run tool calls in parallel. + */ + parallel_tool_calls?: boolean | null; + + /** + * The unique ID of the previous response to the model. Use this to create + * multi-turn conversations. Learn more about + * [conversation state](https://platform.openai.com/docs/guides/conversation-state). + */ + previous_response_id?: string | null; + + /** + * **o-series models only** + * + * Configuration options for + * [reasoning models](https://platform.openai.com/docs/guides/reasoning). + */ + reasoning?: Shared.Reasoning | null; + + /** + * Whether to store the generated model response for later retrieval via API. + */ + store?: boolean | null; + + /** + * If set to true, the model response data will be streamed to the client as it is + * generated using + * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + * See the + * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) + * for more information. + */ + stream?: boolean | null; + + /** + * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + * make the output more random, while lower values like 0.2 will make it more + * focused and deterministic. We generally recommend altering this or `top_p` but + * not both. + */ + temperature?: number | null; + + /** + * Configuration options for a text response from the model. Can be plain text or + * structured JSON data. Learn more: + * + * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) + * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) + */ + text?: ResponseTextConfig; + + /** + * How the model should select which tool (or tools) to use when generating a + * response. See the `tools` parameter to see how to specify which tools the model + * can call. + */ + tool_choice?: ToolChoiceOptions | ToolChoiceTypes | ToolChoiceFunction; + + /** + * An array of tools the model may call while generating a response. You can + * specify which tool to use by setting the `tool_choice` parameter. + * + * The two categories of tools you can provide the model are: + * + * - **Built-in tools**: Tools that are provided by OpenAI that extend the model's + * capabilities, like + * [web search](https://platform.openai.com/docs/guides/tools-web-search) or + * [file search](https://platform.openai.com/docs/guides/tools-file-search). + * Learn more about + * [built-in tools](https://platform.openai.com/docs/guides/tools). + * - **Function calls (custom tools)**: Functions that are defined by you, enabling + * the model to call your own code. Learn more about + * [function calling](https://platform.openai.com/docs/guides/function-calling). + */ + tools?: Array; + + /** + * An alternative to sampling with temperature, called nucleus sampling, where the + * model considers the results of the tokens with top_p probability mass. So 0.1 + * means only the tokens comprising the top 10% probability mass are considered. + * + * We generally recommend altering this or `temperature` but not both. + */ + top_p?: number | null; + + /** + * The truncation strategy to use for the model response. + * + * - `auto`: If the context of this response and previous ones exceeds the model's + * context window size, the model will truncate the response to fit the context + * window by dropping input items in the middle of the conversation. + * - `disabled` (default): If a model response will exceed the context window size + * for a model, the request will fail with a 400 error. + */ + truncation?: 'auto' | 'disabled' | null; + + /** + * A unique identifier representing your end-user, which can help OpenAI to monitor + * and detect abuse. + * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + */ + user?: string; +} + +export interface ResponseCreateParamsNonStreaming + extends ResponseCreateParamsBase { + /** + * If set to true, the model response data will be streamed to the client as it is + * generated using + * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + * See the + * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) + * for more information. + */ + stream?: false | null; +} + +export interface ResponseCreateParamsStreaming + extends ResponseCreateParamsBase { + /** + * If set to true, the model response data will be streamed to the client as it is + * generated using + * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + * See the + * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) + * for more information. + */ + stream: true; +} + +export interface ResponseRetrieveParams { + /** + * Additional fields to include in the response. See the `include` parameter for + * Response creation above for more information. + */ + include?: Array; +} + +export interface ModelResponseDeleteResponse { + id: string; + object: 'response'; + deleted: boolean; +} diff --git a/src/types/requestBody.ts b/src/types/requestBody.ts new file mode 100644 index 0000000..cbc8395 --- /dev/null +++ b/src/types/requestBody.ts @@ -0,0 +1,550 @@ +import { BatchEndpoints } from '../globals'; +import { HookObject } from '../middlewares/hooks/types'; + +/** + * Settings for retrying requests. + * @interface + */ +export interface RetrySettings { + /** The maximum number of retry attempts. */ + attempts: number; + /** The HTTP status codes on which to retry. */ + onStatusCodes: number[]; + /** Whether to use the provider's retry wait. */ + useRetryAfterHeader?: boolean; +} + +export interface CacheSettings { + mode: string; + maxAge?: number; +} + +export enum StrategyModes { + LOADBALANCE = 'loadbalance', + FALLBACK = 'fallback', + SINGLE = 'single', + CONDITIONAL = 'conditional', +} + +interface Strategy { + mode: StrategyModes; + onStatusCodes?: Array; + conditions?: { + query: { + [key: string]: any; + }; + then: string; + }[]; + default?: string; +} + +/** + * Configuration for an AI provider. + * @interface + */ +export interface Options { + /** The name of the provider. */ + provider: string; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + /** The weight of the provider, used for load balancing. */ + weight?: number; + /** The retry settings for the provider. */ + retry?: RetrySettings; + /** The parameters to override in the request. */ + overrideParams?: Params; + /** The actual url used to make llm calls */ + urlToFetch?: string; + /** Azure specific */ + resourceName?: string; + deploymentId?: string; + apiVersion?: string; + adAuth?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureWorkloadClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureAdToken?: string; + azureModelName?: string; + /** Workers AI specific */ + workersAiAccountId?: string; + /** The parameter to set custom base url */ + customHost?: string; + /** The parameter to set list of headers to be forwarded as-is to the provider */ + forwardHeaders?: string[]; + /** provider option index picked based on weight in loadbalance mode */ + index?: number; + cache?: CacheSettings | string; + metadata?: Record; + requestTimeout?: number; + /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ + transformToFormData?: boolean; + /** AWS specific (used for Bedrock and Sagemaker) */ + awsSecretAccessKey?: string; + awsAccessKeyId?: string; + awsSessionToken?: string; + awsRegion?: string; + awsAuthType?: string; + awsRoleArn?: string; + awsExternalId?: string; + awsS3Bucket?: string; + awsS3ObjectKey?: string; + awsBedrockModel?: string; + awsServerSideEncryption?: string; + awsServerSideEncryptionKMSKeyId?: string; + awsService?: string; + foundationModel?: string; + + /** Sagemaker specific */ + amznSagemakerCustomAttributes?: string; + amznSagemakerTargetModel?: string; + amznSagemakerTargetVariant?: string; + amznSagemakerTargetContainerHostname?: string; + amznSagemakerInferenceId?: string; + amznSagemakerEnableExplanations?: string; + amznSagemakerInferenceComponent?: string; + amznSagemakerSessionId?: string; + amznSagemakerModelName?: string; + + /** Stability AI specific */ + stabilityClientId?: string; + stabilityClientUserId?: string; + stabilityClientVersion?: string; + + /** Hugging Face specific */ + huggingfaceBaseUrl?: string; + + /** Google Vertex AI specific */ + vertexRegion?: string; + vertexProjectId?: string; + vertexServiceAccountJson?: Record; + vertexStorageBucketName?: string; + vertexModelName?: string; + vertexBatchEndpoint?: BatchEndpoints; + + // Required for file uploads with google. + filename?: string; + + afterRequestHooks?: HookObject[]; + beforeRequestHooks?: HookObject[]; + defaultInputGuardrails?: HookObject[]; + defaultOutputGuardrails?: HookObject[]; + /** OpenAI specific */ + openaiProject?: string; + openaiOrganization?: string; + openaiBeta?: string; + /** Azure Inference Specific */ + azureApiVersion?: string; + azureFoundryUrl?: string; + azureExtraParameters?: string; + azureDeploymentName?: string; + + /** The parameter to determine if extra non-openai compliant fields should be returned in response */ + strictOpenAiCompliance?: boolean; + + /** Parameter to determine if fim/completions endpoint is to be used */ + mistralFimCompletion?: string; + + /** Anthropic specific headers */ + anthropicBeta?: string; + anthropicVersion?: string; + anthropicApiKey?: string; + + /** Fireworks finetune required fields */ + fireworksAccountId?: string; + fireworksFileLength?: string; + + /** Cortex specific fields */ + snowflakeAccount?: string; + + /** Azure entra scope */ + azureEntraScope?: string; + + // Oracle specific fields + oracleApiVersion?: string; // example: 20160918 + oracleRegion?: string; // example: us-ashburn-1 + oracleCompartmentId?: string; // example: ocid1.compartment.oc1..aaaaaaaab7x77777777777777777 + oracleServingMode?: string; // supported values: ON_DEMAND, DEDICATED + oracleTenancy?: string; // example: ocid1.tenancy.oc1..aaaaaaaab7x77777777777777777 + oracleUser?: string; // example: ocid1.user.oc1..aaaaaaaab7x77777777777777777 + oracleFingerprint?: string; // example: 12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef + oraclePrivateKey?: string; // example: -----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA... + oracleKeyPassphrase?: string; // example: password + + /** Model pricing config */ + modelPricingConfig?: Record; +} + +/** + * Configuration for an AI provider. + * @interface + */ +export interface Targets { + name?: string; + strategy?: Strategy; + /** The name of the provider. */ + provider?: string | undefined; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + /** The weight of the provider, used for load balancing. */ + weight?: number; + /** The retry settings for the provider. */ + retry?: RetrySettings; + /** The parameters to override in the request. */ + overrideParams?: Params; + /** The actual url used to make llm calls */ + urlToFetch?: string; + /** Azure specific */ + resourceName?: string; + deploymentId?: string; + apiVersion?: string; + adAuth?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureModelName?: string; + /** provider option index picked based on weight in loadbalance mode */ + index?: number; + cache?: CacheSettings | string; + targets?: Targets[]; + + /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ + transformToFormData?: boolean; + + defaultInputGuardrails?: HookObject[]; + defaultOutputGuardrails?: HookObject[]; + originalIndex?: number; +} + +/** + * Configuration for handling the request. + * @interface + */ +export interface Config { + /** The mode for handling the request. It can be "single", "fallback", "loadbalance", or "scientist". */ + mode: 'single' | 'fallback' | 'loadbalance' | 'scientist'; + /** The configuration for the provider(s). */ + options: Options[]; + targets?: Targets[]; + cache?: CacheSettings; + retry?: RetrySettings; + strategy?: Strategy; + customHost?: string; +} + +/** + * TODO: make this a union type + * A message content type. + * @interface + */ +export interface ContentType extends PromptCache { + type: string; + text?: string; + thinking?: string; + signature?: string; + image_url?: { + url: string; + detail?: string; + mime_type?: string; + }; + data?: string; + file?: { + file_data?: string; + file_id?: string; + file_name?: string; + file_url?: string; + mime_type?: string; + }; + input_audio?: { + data: string; + format: 'mp3' | 'wav' | string; //defaults to auto + }; +} + +export interface ToolCall { + id: string; + type: string; + function: { + name: string; + arguments: string; + description?: string; + thought_signature?: string; + }; +} + +export enum MESSAGE_ROLES { + SYSTEM = 'system', + USER = 'user', + ASSISTANT = 'assistant', + FUNCTION = 'function', + TOOL = 'tool', + DEVELOPER = 'developer', +} + +export const SYSTEM_MESSAGE_ROLES = ['system', 'developer']; + +export type OpenAIMessageRole = + | 'system' + | 'user' + | 'assistant' + | 'function' + | 'tool' + | 'developer'; + +export interface ContentBlockChunk extends Omit { + index: number; + type?: string; +} + +/** + * A message in the conversation. + * @interface + */ +export interface Message { + /** The role of the message sender. It can be 'system', 'user', 'assistant', or 'function'. */ + role: OpenAIMessageRole; + /** The content of the message. */ + content?: string | ContentType[]; + /** The content blocks of the message. */ + content_blocks?: ContentType[]; + /** The name of the function to call, if any. */ + name?: string; + /** The function call to make, if any. */ + function_call?: any; + tool_calls?: any; + tool_call_id?: string; + citationMetadata?: CitationMetadata; + /** Reasoning details for models that support extended thinking/reasoning. (Gemini) */ + reasoning_details?: any[]; +} + +export interface PromptCache { + cache_control?: { type: 'ephemeral' }; +} + +export interface CitationMetadata { + citationSources?: CitationSource[]; +} + +export interface CitationSource { + startIndex?: number; + endIndex?: number; + uri?: string; + license?: string; +} + +/** + * A JSON schema. + * @interface + */ +export interface JsonSchema { + /** The schema definition, indexed by key. */ + [key: string]: any; +} + +/** + * A function in the conversation. + * @interface + */ +export interface Function { + /** The name of the function. */ + name: string; + /** A description of the function. */ + description?: string; + /** The parameters for the function. */ + parameters?: JsonSchema; + /** Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the parameters field. Only a subset of JSON Schema is supported when strict is true */ + strict?: boolean; + /** + * When true, this tool is not loaded into context initially. + * Claude discovers it via Tool Search Tool on-demand. + * Part of Anthropic's advanced tool use beta features. + */ + defer_loading?: boolean; + /** + * List of tool types that can call this tool programmatically. + * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. + * Part of Anthropic's advanced tool use beta features. + */ + allowed_callers?: string[]; + /** + * Example inputs demonstrating how to use this tool. + * Helps Claude understand usage patterns beyond JSON schema. + * Part of Anthropic's advanced tool use beta features. + */ + input_examples?: Record[]; +} + +export interface ToolChoiceObject { + type: string; + function: { + name: string; + }; +} + +export interface CustomToolChoice { + type: 'custom'; + custom: { + name?: string; + }; +} + +export type ToolChoice = + | ToolChoiceObject + | CustomToolChoice + | 'none' + | 'auto' + | 'required'; + +/** + * A tool in the conversation. + * + * `cache_control` is extended to support for prompt-cache + * + * @interface + */ +export interface Tool extends PromptCache { + /** The name of the function. */ + type: string; + /** A description of the function. */ + function?: Function; + // this is used to support tools like computer, web_search, etc. + [key: string]: any; +} + +/** + * The parameters for the request. + * @interface + */ +export interface Params { + model?: string; + prompt?: string | string[]; + messages?: Message[]; + functions?: Function[]; + function_call?: 'none' | 'auto' | { name: string }; + max_tokens?: number; + max_completion_tokens?: number; + temperature?: number; + top_p?: number; + n?: number; + stream?: boolean; + logprobs?: number; + top_logprobs?: boolean; + echo?: boolean; + stop?: string | string[]; + presence_penalty?: number; + frequency_penalty?: number; + best_of?: number; + logit_bias?: { [key: string]: number }; + user?: string; + context?: string; + examples?: Examples[]; + top_k?: number; + tools?: Tool[]; + tool_choice?: ToolChoice; + reasoning_effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | string; + response_format?: { + type: 'json_object' | 'text' | 'json_schema'; + json_schema?: any; + }; + seed?: number; + store?: boolean; + metadata?: object; + modalities?: string[]; + audio?: { + voice: string; + format: string; + }; + service_tier?: string; + prediction?: { + type: string; + content: + | { + type: string; + text: string; + }[] + | string; + }; + // Google Vertex AI specific + safety_settings?: any; + // Anthropic specific + anthropic_beta?: string; + anthropic_version?: string; + thinking?: { + type?: string; + budget_tokens: number; + }; + // Embeddings specific + dimensions?: number; + parameters?: any; +} + +interface Examples { + input?: Message; + output?: Message; +} + +/** + * The full structure of the request body. + * @interface + */ +interface FullRequestBody { + /** The configuration for handling the request. */ + config: Config; + /** The parameters for the request. */ + params: Params; +} + +/** + * A shortened structure of the request body, with a simpler configuration. + * @interface + */ +export interface ShortConfig { + /** The name of the provider. */ + provider: string; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + cache?: CacheSettings; + retry?: RetrySettings; + resourceName?: string; + deploymentId?: string; + workersAiAccountId?: string; + apiVersion?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureModelName?: string; + customHost?: string; + // Google Vertex AI specific + vertexRegion?: string; + vertexProjectId?: string; +} + +/** + * The shortened structure of the request body. + * @interface + */ +interface ShortRequestBody { + /** The simplified configuration for handling the request. */ + config: ShortConfig; + /** The parameters for the request. */ + params: Params; +} + +/** + * The request body, which can be either a `FullRequestBody` or a `ShortRequestBody`. + * @type + */ +export type RequestBody = FullRequestBody | ShortRequestBody; diff --git a/src/types/responseBody.ts b/src/types/responseBody.ts new file mode 100644 index 0000000..2ff6359 --- /dev/null +++ b/src/types/responseBody.ts @@ -0,0 +1,60 @@ +interface CitationSource { + startIndex?: number; + endIndex?: number; + uri?: string; + license?: string; +} + +interface CitationMetadata { + citationSources?: CitationSource[]; +} + +interface PalmMessage { + content?: string; + author?: string; + citationMetadata?: CitationMetadata; +} + +interface ContentFilter { + reason: 'BLOCKED_REASON_UNSPECIFIED' | 'SAFETY' | 'OTHER'; + message: string; +} + +export interface PalmChatCompleteResponse { + candidates: PalmMessage[]; + messages: PalmMessage[]; + filters: ContentFilter[]; + error?: PalmError; +} + +interface PalmTextOutput { + output: string; + safetyRatings: safetyRatings[]; +} + +interface safetyRatings { + category: + | 'HARM_CATEGORY_DEROGATORY' + | 'HARM_CATEGORY_TOXICITY' + | 'HARM_CATEGORY_VIOLENCE' + | 'HARM_CATEGORY_SEXUAL' + | 'HARM_CATEGORY_MEDICAL' + | 'HARM_CATEGORY_DANGEROUS'; + probability: 'NEGLIGIBLE' | 'LOW' | 'HIGH'; +} + +interface PalmFilter { + reason: 'OTHER'; +} + +interface PalmError { + code: number; + message: string; + status: string; +} + +export interface PalmCompleteResponse { + candidates: PalmTextOutput[]; + filters: PalmFilter[]; + error?: PalmError; +} diff --git a/src/types/shared.ts b/src/types/shared.ts new file mode 100644 index 0000000..3928c27 --- /dev/null +++ b/src/types/shared.ts @@ -0,0 +1,119 @@ +/** + * A filter used to compare a specified attribute key to a given value using a + * defined comparison operation. + */ +export interface ComparisonFilter { + /** + * The key to compare against the value. + */ + key: string; + + /** + * Specifies the comparison operator: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`. + * + * - `eq`: equals + * - `ne`: not equal + * - `gt`: greater than + * - `gte`: greater than or equal + * - `lt`: less than + * - `lte`: less than or equal + */ + type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; + + /** + * The value to compare against the attribute key; supports string, number, or + * boolean types. + */ + value: string | number | boolean; +} + +/** + * Combine multiple filters using `and` or `or`. + */ +export interface CompoundFilter { + /** + * Array of filters to combine. Items can be `ComparisonFilter` or + * `CompoundFilter`. + */ + filters: Array; + + /** + * Type of operation: `and` or `or`. + */ + type: 'and' | 'or'; +} + +/** + * Set of 16 key-value pairs that can be attached to an object. This can be useful + * for storing additional information about the object in a structured format, and + * querying for objects via API or the dashboard. + * + * Keys are strings with a maximum length of 64 characters. Values are strings with + * a maximum length of 512 characters. + */ +export type Metadata = Record; + +/** + * **o-series models only** + * + * Configuration options for + * [reasoning models](https://platform.openai.com/docs/guides/reasoning). + */ +export interface Reasoning { + /** + * **o-series models only** + * + * Constrains effort on reasoning for + * [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + * supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + * result in faster responses and fewer tokens used on reasoning in a response. + */ + effort: ReasoningEffort | null; + + /** + * **o-series models only** + * + * A summary of the reasoning performed by the model. This can be useful for + * debugging and understanding the model's reasoning process. One of `concise` or + * `detailed`. + */ + generate_summary?: 'concise' | 'detailed' | null; +} + +/** + * **o-series models only** + * + * Constrains effort on reasoning for + * [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + * supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + * result in faster responses and fewer tokens used on reasoning in a response. + */ +export type ReasoningEffort = 'low' | 'medium' | 'high' | null; + +/** + * Default response format. Used to generate text responses. + */ +export interface ResponseFormatText { + /** + * The type of response format being defined. Always `text`. + */ + type: 'text'; +} + +/** + * JSON object response format. An older method of generating JSON responses. Using + * `json_schema` is recommended for models that support it. Note that the model + * will not generate JSON without a system or user message instructing it to do so. + */ +export interface ResponseFormatJSONObject { + /** + * The type of response format being defined. Always `json_object`. + */ + type: 'json_object'; +} + +export interface CursorPageParams { + after?: string; + + limit?: number; +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..a6a4541 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,117 @@ +import { + ANTHROPIC, + COHERE, + GOOGLE, + GOOGLE_VERTEX_AI, + PERPLEXITY_AI, + DEEPINFRA, + SAMBANOVA, + BEDROCK, + BYTEZ, +} from './globals'; +import { Params } from './types/requestBody'; + +export const getStreamModeSplitPattern = ( + proxyProvider: string, + requestURL: string +) => { + let splitPattern: SplitPatternType = '\n\n'; + + if (proxyProvider === ANTHROPIC && requestURL.endsWith('/complete')) { + splitPattern = '\r\n\r\n'; + } + + if (proxyProvider === COHERE) { + splitPattern = requestURL.includes('/chat') ? '\n\n' : '\n'; + } + + if (proxyProvider === GOOGLE) { + splitPattern = '\r\n'; + } + + // In Vertex Anthropic and LLama have \n\n as the pattern only Gemini has \r\n\r\n + if ( + proxyProvider === GOOGLE_VERTEX_AI && + requestURL.includes('/publishers/google') + ) { + splitPattern = '\r\n\r\n'; + } + + if (proxyProvider === PERPLEXITY_AI) { + splitPattern = '\r\n\r\n'; + } + + if (proxyProvider === DEEPINFRA) { + splitPattern = '\n'; + } + + if (proxyProvider === SAMBANOVA) { + splitPattern = '\n'; + } + + if (proxyProvider === BYTEZ) { + splitPattern = ' '; + } + + return splitPattern; +}; +export type SplitPatternType = '\n\n' | '\r\n\r\n' | '\n' | '\r\n' | ' '; + +export const getStreamingMode = ( + reqBody: Params, + provider: string, + requestUrl: string +) => { + if ( + [GOOGLE, GOOGLE_VERTEX_AI].includes(provider) && + requestUrl.indexOf('stream') > -1 + ) { + return true; + } + if ( + provider === BEDROCK && + (requestUrl.indexOf('invoke-with-response-stream') > -1 || + requestUrl.indexOf('converse-stream') > -1) + ) { + return true; + } + return reqBody.stream; +}; + +export function convertKeysToCamelCase( + obj: Record, + parentKeysToPreserve: string[] = [] +): Record { + if (typeof obj !== 'object' || obj === null) { + return obj; // Return unchanged for non-objects or null + } + + if (Array.isArray(obj)) { + // If it's an array, recursively convert each element + return obj.map((item) => + convertKeysToCamelCase(item, parentKeysToPreserve) + ); + } + + return Object.keys(obj).reduce((result: any, key: string) => { + const value = obj[key]; + const camelCaseKey = toCamelCase(key); + const isParentKeyToPreserve = parentKeysToPreserve.includes(key); + if (typeof value === 'object' && !isParentKeyToPreserve) { + // Recursively convert child objects + result[camelCaseKey] = convertKeysToCamelCase( + value, + parentKeysToPreserve + ); + } else { + // Add key in camelCase to the result + result[camelCaseKey] = value; + } + + return result; + }, {}); + + function toCamelCase(snakeCase: string): string { + return snakeCase.replace(/(_\w)/g, (match) => match[1].toUpperCase()); + } +} diff --git a/src/utils/CryptoUtils.ts b/src/utils/CryptoUtils.ts new file mode 100644 index 0000000..b5e6ed0 --- /dev/null +++ b/src/utils/CryptoUtils.ts @@ -0,0 +1,152 @@ +// Crypto utilities that work in both Node.js and Cloudflare Workers +export class CryptoUtils { + /** + * Normalize a PEM key that might be missing newlines + */ + private static normalizePemKey(pemKey: string): string { + // Remove all whitespace first + let normalized = pemKey.trim().replace(/\s+/g, ''); + + // Check for BEGIN/END markers + const beginMarkers = [ + '-----BEGINPRIVATEKEY-----', + '-----BEGINRSAPRIVATEKEY-----', + '-----BEGINENCRYPTEDPRIVATEKEY-----', + ]; + + const endMarkers = [ + '-----ENDPRIVATEKEY-----', + '-----ENDRSAPRIVATEKEY-----', + '-----ENDENCRYPTEDPRIVATEKEY-----', + ]; + + let beginMarker = ''; + let endMarker = ''; + let keyContent = normalized; + + // Find which markers are present + for (let i = 0; i < beginMarkers.length; i++) { + if (normalized.includes(beginMarkers[i])) { + beginMarker = beginMarkers[i]; + endMarker = endMarkers[i]; + // Extract content between markers + const startIdx = normalized.indexOf(beginMarker) + beginMarker.length; + const endIdx = normalized.indexOf(endMarker); + keyContent = normalized.substring(startIdx, endIdx); + break; + } + } + + // If no markers found, assume the whole thing is the key content + if (!beginMarker) { + beginMarker = '-----BEGINPRIVATEKEY-----'; + endMarker = '-----ENDPRIVATEKEY-----'; + } + + // Reformat with proper newlines (64 chars per line is PEM standard) + const formattedContent = + keyContent.match(/.{1,64}/g)?.join('\n') || keyContent; + + // Reconstruct with proper spacing + const properBegin = beginMarker + .replace('-----BEGIN', '-----BEGIN ') + .replace('KEY-----', ' KEY-----'); + const properEnd = endMarker + .replace('-----END', '-----END ') + .replace('KEY-----', ' KEY-----'); + + return `${properBegin}\n${formattedContent}\n${properEnd}`; + } + + private static async importPrivateKey( + pemKey: string, + passphrase?: string + ): Promise { + // Normalize the key first + const normalizedKey = this.normalizePemKey(pemKey); + + // Remove PEM headers and decode base64 + const pemHeader = '-----BEGIN'; + const pemFooter = '-----END'; + const pemContents = normalizedKey + .split('\n') + .filter((line) => !line.includes(pemHeader) && !line.includes(pemFooter)) + .join(''); + + const binaryDer = this.base64ToArrayBuffer(pemContents); + + // Import the key using Web Crypto API + try { + return await crypto.subtle.importKey( + 'pkcs8', + binaryDer, + { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-256', + }, + false, + ['sign'] + ); + } catch (error) { + throw new Error( + `Failed to import private key. Ensure it's in PKCS8 format. Use: openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in key.pem -out key_pkcs8.pem` + ); + } + } + + private static base64ToArrayBuffer(base64: string): ArrayBuffer { + const binaryString = + typeof atob !== 'undefined' + ? atob(base64) + : Buffer.from(base64, 'base64').toString('binary'); + + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes.buffer; + } + + private static arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + let binary = ''; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + return typeof btoa !== 'undefined' + ? btoa(binary) + : Buffer.from(binary, 'binary').toString('base64'); + } + + static async sign(privateKey: CryptoKey, data: string): Promise { + const encoder = new TextEncoder(); + const dataBuffer = encoder.encode(data); + + const signature = await crypto.subtle.sign( + 'RSASSA-PKCS1-v1_5', + privateKey, + dataBuffer + ); + + return this.arrayBufferToBase64(signature); + } + + static async sha256(data: string): Promise { + const encoder = new TextEncoder(); + const dataBuffer = encoder.encode(data); + const hashBuffer = await crypto.subtle.digest('SHA-256', dataBuffer); + return this.arrayBufferToBase64(hashBuffer); + } + + static async loadPrivateKey( + pemKey: string, + passphrase?: string + ): Promise { + if (passphrase) { + console.warn( + 'Key passphrase provided but not supported in Web Crypto API. Please use an unencrypted key.' + ); + } + return this.importPrivateKey(pemKey, passphrase); + } +} diff --git a/src/utils/env.ts b/src/utils/env.ts new file mode 100644 index 0000000..22c6c95 --- /dev/null +++ b/src/utils/env.ts @@ -0,0 +1,146 @@ +import { Context } from 'hono'; +import { env, getRuntimeKey } from 'hono/adapter'; + +const isNodeInstance = getRuntimeKey() == 'node'; +let path: any; +let fs: any; +if (isNodeInstance) { + path = await import('path'); + fs = await import('fs'); +} + +export function getValueOrFileContents(value?: string, ignore?: boolean) { + if (!value || ignore) return value; + + try { + // Check if value looks like a file path + if ( + value.startsWith('/') || + value.startsWith('./') || + value.startsWith('../') + ) { + // Resolve the path (handle relative paths) + const resolvedPath = path.resolve(value); + + // Check if file exists + if (fs.existsSync(resolvedPath)) { + // File exists, read and return its contents + return fs.readFileSync(resolvedPath, 'utf8').trim(); + } + } + + // If not a file path or file doesn't exist, return value as is + return value; + } catch (error: any) { + console.log(`Error reading file at ${value}: ${error.message}`); + // Return the original value if there's an error + return value; + } +} + +const nodeEnv = { + NODE_ENV: getValueOrFileContents(process.env.NODE_ENV, true), + PORT: getValueOrFileContents(process.env.PORT) || 8787, + + TLS_KEY_PATH: getValueOrFileContents(process.env.TLS_KEY_PATH, true), + TLS_CERT_PATH: getValueOrFileContents(process.env.TLS_CERT_PATH, true), + TLS_CA_PATH: getValueOrFileContents(process.env.TLS_CA_PATH, true), + + AWS_ACCESS_KEY_ID: getValueOrFileContents(process.env.AWS_ACCESS_KEY_ID), + AWS_SECRET_ACCESS_KEY: getValueOrFileContents( + process.env.AWS_SECRET_ACCESS_KEY + ), + AWS_SESSION_TOKEN: getValueOrFileContents(process.env.AWS_SESSION_TOKEN), + AWS_ROLE_ARN: getValueOrFileContents(process.env.AWS_ROLE_ARN), + AWS_PROFILE: getValueOrFileContents(process.env.AWS_PROFILE, true), + AWS_WEB_IDENTITY_TOKEN_FILE: getValueOrFileContents( + process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + true + ), + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: getValueOrFileContents( + process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI, + true + ), + AWS_ASSUME_ROLE_ACCESS_KEY_ID: getValueOrFileContents( + process.env.AWS_ASSUME_ROLE_ACCESS_KEY_ID + ), + AWS_ASSUME_ROLE_SECRET_ACCESS_KEY: getValueOrFileContents( + process.env.AWS_ASSUME_ROLE_SECRET_ACCESS_KEY + ), + AWS_ASSUME_ROLE_REGION: getValueOrFileContents( + process.env.AWS_ASSUME_ROLE_REGION + ), + AWS_REGION: getValueOrFileContents(process.env.AWS_REGION), + AWS_ENDPOINT_DOMAIN: getValueOrFileContents(process.env.AWS_ENDPOINT_DOMAIN), + AWS_IMDS_V1: getValueOrFileContents(process.env.AWS_IMDS_V1), + + AZURE_AUTH_MODE: getValueOrFileContents(process.env.AZURE_AUTH_MODE), + AZURE_ENTRA_CLIENT_ID: getValueOrFileContents( + process.env.AZURE_ENTRA_CLIENT_ID + ), + AZURE_ENTRA_CLIENT_SECRET: getValueOrFileContents( + process.env.AZURE_ENTRA_CLIENT_SECRET + ), + AZURE_ENTRA_TENANT_ID: getValueOrFileContents( + process.env.AZURE_ENTRA_TENANT_ID + ), + AZURE_MANAGED_CLIENT_ID: getValueOrFileContents( + process.env.AZURE_MANAGED_CLIENT_ID + ), + AZURE_MANAGED_VERSION: getValueOrFileContents( + process.env.AZURE_MANAGED_VERSION + ), + AZURE_IDENTITY_ENDPOINT: getValueOrFileContents( + process.env.IDENTITY_ENDPOINT, + true + ), + AZURE_MANAGED_IDENTITY_HEADER: getValueOrFileContents( + process.env.IDENTITY_HEADER + ), + AZURE_AUTHORITY_HOST: getValueOrFileContents( + process.env.AZURE_AUTHORITY_HOST + ), + AZURE_TENANT_ID: getValueOrFileContents(process.env.AZURE_TENANT_ID), + AZURE_CLIENT_ID: getValueOrFileContents(process.env.AZURE_CLIENT_ID), + AZURE_FEDERATED_TOKEN_FILE: getValueOrFileContents( + process.env.AZURE_FEDERATED_TOKEN_FILE + ), + + SSE_ENCRYPTION_TYPE: getValueOrFileContents(process.env.SSE_ENCRYPTION_TYPE), + KMS_KEY_ID: getValueOrFileContents(process.env.KMS_KEY_ID), + KMS_BUCKET_KEY_ENABLED: getValueOrFileContents( + process.env.KMS_BUCKET_KEY_ENABLED + ), + KMS_ENCRYPTION_CONTEXT: getValueOrFileContents( + process.env.KMS_ENCRYPTION_CONTEXT + ), + KMS_ENCRYPTION_ALGORITHM: getValueOrFileContents( + process.env.KMS_ENCRYPTION_ALGORITHM + ), + KMS_ENCRYPTION_CUSTOMER_KEY: getValueOrFileContents( + process.env.KMS_ENCRYPTION_CUSTOMER_KEY + ), + KMS_ENCRYPTION_CUSTOMER_KEY_MD5: getValueOrFileContents( + process.env.KMS_ENCRYPTION_CUSTOMER_KEY_MD5 + ), + KMS_ROLE_ARN: getValueOrFileContents(process.env.KMS_ROLE_ARN), + + HTTP_PROXY: getValueOrFileContents(process.env.HTTP_PROXY), + HTTPS_PROXY: getValueOrFileContents(process.env.HTTPS_PROXY), + + APM_LOGGER: getValueOrFileContents(process.env.APM_LOGGER), + + TRUSTED_CUSTOM_HOSTS: getValueOrFileContents( + process.env.TRUSTED_CUSTOM_HOSTS + ), +}; + +export const Environment = (c?: Context) => { + if (isNodeInstance) { + return nodeEnv; + } + if (c) { + return env(c); + } + return {}; +}; diff --git a/src/utils/misc.ts b/src/utils/misc.ts new file mode 100644 index 0000000..9c14823 --- /dev/null +++ b/src/utils/misc.ts @@ -0,0 +1,21 @@ +import { Context } from 'hono'; +import { getRuntimeKey } from 'hono/adapter'; + +export function toSnakeCase(str: string) { + return str + .replace(/([a-z])([A-Z])/g, '$1_$2') // Handle camelCase and PascalCase + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') // Handle acronyms + .replace(/[^a-zA-Z0-9]+/g, '_') // Replace special characters with '_' + .replace(/_+/g, '_') // Merge multiple underscores + .toLowerCase(); +} + +export const addBackgroundTask = ( + c: Context, + promise: Promise +) => { + if (getRuntimeKey() === 'workerd') { + c.executionCtx.waitUntil(promise); + } + // in other runtimes, the promise resolves in the background +}; diff --git a/start-test.js b/start-test.js new file mode 100644 index 0000000..b4ba7be --- /dev/null +++ b/start-test.js @@ -0,0 +1,22 @@ +import { spawn } from 'node:child_process'; + +console.log('Starting the application...'); + +const app = spawn('node', ['build/start-server.js', '--headless'], { + stdio: 'inherit', +}); + +// Listen for errors when spawning the process +app.on('exit', (err) => { + console.error('Failed to start the app:', err); + process.exit(1); // Exit with a failure code if there is an error +}); + +// Listen for when the process starts +app.on('spawn', () => { + console.log('App started successfully'); + setTimeout(() => { + app.kill(); + process.exit(0); + }, 3000); +}); diff --git a/tests/integration/src/handlers/.creds.example.json b/tests/integration/src/handlers/.creds.example.json new file mode 100644 index 0000000..176b612 --- /dev/null +++ b/tests/integration/src/handlers/.creds.example.json @@ -0,0 +1,19 @@ +{ + "azure": { + "apiKey": "YOUR_AZURE_API_KEY" + }, + "aws": { + "accessKeyId": "YOUR_AWS_ACCESS_KEY_ID", + "secretAccessKey": "YOUR_AWS_SECRET_ACCESS_KEY", + "region": "YOUR_AWS_REGION" + }, + "openai": { + "apiKey": "YOUR_OPENAI_API_KEY" + }, + "anthropic": { + "apiKey": "YOUR_ANTHROPIC_API_KEY" + }, + "portkey": { + "apiKey": "YOUR_PORTKEY_API_KEY" + } +} diff --git a/tests/integration/src/handlers/requestBuilder.ts b/tests/integration/src/handlers/requestBuilder.ts new file mode 100644 index 0000000..c32f1a9 --- /dev/null +++ b/tests/integration/src/handlers/requestBuilder.ts @@ -0,0 +1,204 @@ +import { Portkey } from 'portkey-ai'; +import { readFileSync } from 'fs'; +import { join } from 'path'; + +const creds = JSON.parse(readFileSync(join(__dirname, '.creds.json'), 'utf8')); + +export class RequestBuilder { + private requestBody: Record | FormData = {}; + private requestHeaders: Record = {}; + private _method: string = 'POST'; + private _client: Portkey; + + constructor() { + this.requestBody = { + model: 'claude-3-5-sonnet-20240620', + messages: [{ role: 'user', content: 'Hey' }], + max_tokens: 10, + }; + this.requestHeaders = { + 'Content-Type': 'application/json', + 'x-portkey-provider': 'anthropic', + Authorization: `Bearer ${creds.anthropic.apiKey}`, + 'x-portkey-api-key': creds.portkey.apiKey, + }; + this._method = 'POST'; + this._client = new Portkey({ + baseURL: 'http://localhost:8787/v1', + config: { + provider: 'anthropic', + api_key: creds.anthropic.apiKey, + }, + }); + } + + useGet() { + this._method = 'GET'; + return this; + } + + get client() { + return this._client; + } + + get options() { + const _options: any = { + method: this._method, + body: + this.requestBody instanceof FormData + ? this.requestBody + : JSON.stringify(this.requestBody), + headers: { ...this.requestHeaders }, + }; + + if (this.requestBody instanceof FormData) { + const { ['Content-Type']: _, ...restHeaders } = this.requestHeaders; + _options.headers = restHeaders; + } + + if (this._method === 'GET') { + delete _options.body; + } + return _options; + } + + model(model: string) { + if (this.requestBody instanceof FormData) { + throw new Error('Model cannot be set for FormData'); + } + this.requestBody.model = model; + return this; + } + + messages(messages: any[]) { + if (this.requestBody instanceof FormData) { + throw new Error('Messages cannot be set for FormData'); + } + this.requestBody.messages = messages; + return this; + } + + maxTokens(maxTokens: number) { + if (this.requestBody instanceof FormData) { + throw new Error('Max tokens cannot be set for FormData'); + } + this.requestBody.max_tokens = maxTokens; + return this; + } + + stream(stream: boolean) { + if (this.requestBody instanceof FormData) { + throw new Error('Stream cannot be set for FormData'); + } + this.requestBody.stream = stream; + return this; + } + + provider(provider: string) { + this.requestHeaders['x-portkey-provider'] = provider; + if (provider === 'openai') { + this.apiKey(creds.openai.apiKey); + } else if (provider === 'anthropic') { + this.apiKey(creds.anthropic.apiKey); + } + return this; + } + + providerHeaders(providerHeaders: Record) { + // for each key, switch all underscores to hyphens + // and prepend with x-portkey- + const _providerHeaders: any = {}; + for (const [key, value] of Object.entries(providerHeaders)) { + _providerHeaders[`x-portkey-${key.replace(/_/g, '-')}`] = value; + } + this.requestHeaders = { + ...this.requestHeaders, + ..._providerHeaders, + }; + return this; + } + + addHeaders(headers: Record) { + this.requestHeaders = { + ...this.requestHeaders, + ...headers, + }; + return this; + } + + apiKey(apiKey: string) { + if (apiKey) { + this.requestHeaders['Authorization'] = `Bearer ${apiKey}`; + } else { + delete this.requestHeaders['Authorization']; + } + return this; + } + + body(body: Record | FormData) { + this.requestBody = body; + // If we're switching to FormData we must remove any stale JSON content-type header + if (body instanceof FormData) { + delete this.requestHeaders['Content-Type']; + } + return this; + } + + config(config: any) { + this._client.config = config; + // Create headers for this config + const configHeader = { + 'x-portkey-config': JSON.stringify(config), + }; + this.requestHeaders = { + ...this.requestHeaders, + ...configHeader, + }; + return this; + } +} + +export class URLBuilder { + private _url: string = 'http://localhost:8787/v1'; + + constructor() {} + + get url() { + return this._url; + } + + endpoint(endpoint: string) { + this._url = `${this._url}/${endpoint}`; + return this; + } + + chat() { + this.endpoint('chat/completions'); + return this._url; + } + + files() { + this.endpoint('files'); + return this._url; + } + + transcription() { + this.endpoint('audio/transcriptions'); + return this._url; + } + + images() { + this.endpoint('images/generations'); + return this._url; + } + + path(path: string) { + this.endpoint(path); + return this._url; + } + + clear() { + this._url = 'http://localhost:8787/v1'; + return this; + } +} diff --git a/tests/integration/src/handlers/round1.mp3 b/tests/integration/src/handlers/round1.mp3 new file mode 100644 index 0000000..61ee442 Binary files /dev/null and b/tests/integration/src/handlers/round1.mp3 differ diff --git a/tests/integration/src/handlers/speech2.mp3 b/tests/integration/src/handlers/speech2.mp3 new file mode 100644 index 0000000..5130575 Binary files /dev/null and b/tests/integration/src/handlers/speech2.mp3 differ diff --git a/tests/integration/src/handlers/test.txt b/tests/integration/src/handlers/test.txt new file mode 100644 index 0000000..a8a9406 --- /dev/null +++ b/tests/integration/src/handlers/test.txt @@ -0,0 +1 @@ +this is a test \ No newline at end of file diff --git a/tests/integration/src/handlers/tryPost.test.ts b/tests/integration/src/handlers/tryPost.test.ts new file mode 100644 index 0000000..1325f89 --- /dev/null +++ b/tests/integration/src/handlers/tryPost.test.ts @@ -0,0 +1,574 @@ +import { readFileSync } from 'fs'; +import { RequestBuilder, URLBuilder } from './requestBuilder'; +import { join } from 'path'; + +let requestBuilder: RequestBuilder, urlBuilder: URLBuilder; + +// Gateway tests +describe('core functionality', () => { + beforeEach(() => { + requestBuilder = new RequestBuilder(); + urlBuilder = new URLBuilder(); + }); + + it('should handle a simple chat completion request', async () => { + const url = urlBuilder.chat(); + const options = requestBuilder.model('claude-3-5-sonnet-20240620').options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(data.choices[0].message.content).toBeDefined(); + expect(response.status).toBe(200); + }); + + it('should handle a simple chat completion request with stream', async () => { + const url = urlBuilder.chat(); + const options = requestBuilder + .model('claude-3-5-sonnet-20240620') + .stream(true).options; + + const response = await fetch(url, options); + + //expect response to be a stream + expect(response.body).toBeDefined(); + expect(response.status).toBe(200); + expect(response.body).toBeInstanceOf(ReadableStream); + }); + + it('should handle binary file uploads with FormData', async () => { + const formData = new FormData(); + // Append a random file to formData + formData.append( + 'file', + new Blob([readFileSync('./src/handlers/tests/test.txt')]), + 'test.txt' + ); + formData.append('purpose', 'assistants'); + + const url = urlBuilder.files(); + const options = requestBuilder.provider('openai').body(formData).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.object).toBe('file'); + expect(data.purpose).toBe('assistants'); + }); + + it('should handle audio transcription with ArrayBuffer', async () => { + try { + const formData = new FormData(); + formData.append( + 'file', + new Blob([readFileSync('./src/handlers/tests/speech2.mp3')]), + 'speech2.mp3' + ); + formData.append('model', 'gpt-4o-transcribe'); + + const url = urlBuilder.transcription(); + const options = requestBuilder.provider('openai').body(formData).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.text).toBe('Today is a wonderful day to play.'); + } catch (error) { + expect(error).toBeUndefined(); + } + }); + + it('should handle image generation requests', async () => { + const url = urlBuilder.images(); + const options = requestBuilder.provider('openai').body({ + prompt: 'A beautiful sunset over a calm ocean', + n: 1, + size: '1024x1024', + model: 'dall-e-3', + }).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.data[0].b64_json || data.data[0].url).toBeDefined(); + // console.log(data.data[0].b64_json || data.data[0].url); + }); + + it('should handle proxy requests with custom paths', async () => { + const url = urlBuilder.path('models'); + const options = requestBuilder.provider('openai').useGet().options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.object).toBe('list'); + + // console.log(data); + }); + + // TODO: some more difficult proxy paths with different file types here. +}); + +describe.skip('tryPost-provider-specific', () => { + beforeEach(() => { + requestBuilder = new RequestBuilder(); + urlBuilder = new URLBuilder(); + }); + + it('should handle Azure OpenAI with resource names and deployment IDs', async () => { + // Verify Azure URL construction and headers + const url = urlBuilder.chat(); + const creds = JSON.parse( + readFileSync(join(__dirname, '.creds.json'), 'utf8') + ); + const options = requestBuilder + .provider('azure-openai') + .apiKey(creds.azure.apiKey) + .providerHeaders({ + resource_name: 'portkey', + deployment_id: 'turbo-16k', + api_version: '2023-03-15-preview', + }).options; + + const response = await fetch(url, options); + if (response.status !== 200) { + console.log(await response.text()); + } + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.choices[0].message.content).toBeDefined(); + }); + + it('should handle AWS Bedrock with SigV4 authentication', async () => { + // Verify AWS auth headers are generated + const url = urlBuilder.chat(); + const creds = JSON.parse( + readFileSync(join(__dirname, '.creds.json'), 'utf8') + ); + const options = requestBuilder + .provider('bedrock') + .model('cohere.command-r-v1:0') + .apiKey('') + .providerHeaders({ + aws_access_key_id: creds.aws.accessKeyId, + aws_secret_access_key: creds.aws.secretAccessKey, + aws_region: creds.aws.region, + }).options; + + const response = await fetch(url, options); + if (response.status !== 200) { + console.log(await response.text()); + } + const data: any = await response.json(); + // console.log(data); + + expect(response.status).toBe(200); + expect(data.choices[0].message.content).toBeDefined(); + }); + + it('should handle Google Vertex AI with service account auth', async () => { + // Verify Vertex AI auth and URL construction + }); + + it('should handle provider with custom request handler', async () => { + // Verify custom handlers bypass normal transformation + }); + + it.only('should handle invalid provider gracefully', async () => { + // Verify error when provider not found + const url = urlBuilder.chat(); + const options = requestBuilder + .provider('non-existent-provider') + .apiKey('some-key') + .messages([{ role: 'user', content: 'Hello' }]).options; + + const response = await fetch(url, options); + const error: any = await response.json(); + + console.log(error); + + expect(response.status).toBe(400); + expect(error.status).toBe('failure'); + expect(error.message).toMatch(/Invalid provider/i); + }); +}); + +describe('tryPost-error-handling', () => { + beforeEach(() => { + requestBuilder = new RequestBuilder(); + urlBuilder = new URLBuilder(); + }); + + it('should through a 446 if after request guardrail fails', async () => { + const url = urlBuilder.chat(); + const options = requestBuilder.config({ + guardrails: { enabled: true }, + }); + }); + + it('should retry when status code is set', async () => { + // Verify retry logic with default retry config + const url = urlBuilder.chat(); + const options = requestBuilder.apiKey('wrong api key').config({ + retry: { attempts: 2, on_status_codes: [401] }, + }).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.headers.get('x-portkey-retry-attempt-count')).toBe('-1'); + + expect(response.status).toBe(401); + expect(data.error.message).toMatch(/invalid/i); + }); + + it('should handle network timeouts with requestTimeout', async () => { + // Verify timeout cancels request + const url = urlBuilder.chat(); + const options = requestBuilder.config({ + request_timeout: 500, + }).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + console.log(data); + console.log(response.status); + + expect(response.status).toBe(408); + expect(data.error.message).toMatch(/timeout/i); + }); +}); + +describe('tryPost-hooks-and-guardrails', () => { + const containsGuardrail = ( + words: string[] = ['word1', 'word2'], + operator: string = 'any', + not: boolean = false, + deny: boolean = false, + async: boolean = false + ) => ({ + id: 'guardrail-1', + async: async, + type: 'guardrail', + deny: deny, + checks: [ + { + id: 'default.contains', + parameters: { + words: words, + operator: operator, + not: not, + }, + }, + ], + }); + + const exaGuardrail = () => ({ + id: 'guardrail-exa', + type: 'guardrail', + deny: false, + checks: [ + { + id: 'exa.online', + parameters: { + numResults: 3, + credentials: { + apiKey: 'ae56af0a-7d05-4595-a228-436fd36476f9', + }, + prefix: '\nHere are some web search results:\n', + suffix: '\n---', + }, + }, + ], + }); + + const invalidGuardrail = () => ({ + id: 'guardrail-invalid', + type: 'guardrail', + deny: false, + checks: [ + { + id: 'invalid.check.that.does.not.exist', + parameters: { + // Invalid parameters that should cause an error + invalidParam: null, + missingRequired: undefined, + }, + }, + ], + }); + + beforeEach(() => { + requestBuilder = new RequestBuilder(); + urlBuilder = new URLBuilder(); + }); + + it('should execute before request hooks and allow request', async () => { + // Verify hooks run and pass + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + before_request_hooks: [ + containsGuardrail(['word1', 'word2'], 'any', false), + ], + }) + .messages([ + { + role: 'user', + content: + 'adding some text before this word1 and adding some text after', + }, + ]).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + // console.log(data.hook_results.before_request_hooks[0].checks[0]); + + expect(response.status).toBe(200); + expect(data.hook_results.before_request_hooks[0].checks[0]).toBeDefined(); + }); + + it('should block request when before request hook denies', async () => { + // Verify 446 response with hook results + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + before_request_hooks: [ + containsGuardrail(['word1', 'word2'], 'none', false, true), + ], + }) + .messages([ + { + role: 'user', + content: + 'adding some text before this word1 and adding some text after', + }, + ]).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(446); + expect(data.hook_results.before_request_hooks[0].checks[0]).toBeDefined(); + }); + + it('should transform request body via before request hooks', async () => { + // Critical: Verify hook transformations work + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + before_request_hooks: [exaGuardrail()], + }) + .messages([ + { + role: 'user', + content: + 'Based on the web search results, who was the chief minister of Delhi in May 2025? reply with name only.', + }, + ]).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.choices[0].message.content).toBeDefined(); + expect(data.choices[0].message.content).toMatch(/Rekha/i); + }); + + it('should execute after request hooks on response', async () => { + // Verify response hooks run + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + after_request_hooks: [ + containsGuardrail(['word1', 'word2'], 'any', false), + ], + }) + .messages([ + { + role: 'user', + content: "Reply with any of 'word1' or 'word2' and nothing else.", + }, + ]).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.hook_results.after_request_hooks[0].checks[0]).toBeDefined(); + }); + + it('should handle failing after request hooks with retry', async () => { + // Verify retry when after hooks fail + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + after_request_hooks: [ + containsGuardrail(['word1', 'word2'], 'none', false, true), + ], + retry: { + attempts: 2, + on_status_codes: [446], + }, + }) + .messages([ + { + role: 'user', + content: "Reply with any of 'word1' or 'word2' and nothing else.", + }, + ]).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(446); + expect(response.headers.get('x-portkey-retry-attempt-count')).toBe('-1'); + expect(data.hook_results.after_request_hooks[0].checks[0]).toBeDefined(); + }); + + it('should include hook results in cached responses', async () => { + // Verify cache includes hook execution results + }); + + it('should handle async hooks without blocking', async () => { + // Verify async hooks don't block response + const url = urlBuilder.chat(); + const options = requestBuilder.config({ + before_request_hooks: [ + containsGuardrail(['word1', 'word2'], 'all', false, true, true), + ], + }).options; + + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(data.hook_results).toBeUndefined(); + }); +}); + +describe('tryPost-caching', () => { + beforeEach(() => { + requestBuilder = new RequestBuilder(); + urlBuilder = new URLBuilder(); + }); + + it('should cache successful responses when cache mode is simple', async () => { + // Verify cache storage and key generation + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + cache: { mode: 'simple' }, + }) + .messages([ + { role: 'user', content: 'Hello' + new Date().getTime() }, + ]).options; + + // Store in cache + const nonCachedResponse = await fetch(url, options); + const nonCachedData: any = await nonCachedResponse.json(); + + expect(nonCachedResponse.status).toBe(200); + expect(nonCachedResponse.headers.get('x-portkey-cache-status')).toBe( + 'MISS' + ); + + // Get from cache + const response = await fetch(url, options); + const data: any = await response.json(); + + expect(response.status).toBe(200); + expect(response.headers.get('x-portkey-cache-status')).toBe('HIT'); + expect(data.choices[0].message.content).toBeDefined(); + }); + + it('should not cache file upload endpoints', async () => { + // Verify non-cacheable endpoints skip cache + const formData = new FormData(); + // Append a random file to formData + formData.append( + 'file', + new Blob([readFileSync('./src/handlers/tests/test.txt')]), + 'test.txt' + ); + formData.append('purpose', 'assistants'); + + const url = urlBuilder.files(); + const options = requestBuilder.provider('openai').body(formData).options; + + const response = await fetch(url, options); + + expect(response.headers.get('x-portkey-cache-status')).toBe('DISABLED'); + }); + + it('should respect cache TTL when configured', async () => { + // Verify maxAge is passed to cache function + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + cache: { mode: 'simple', maxAge: 5000 }, + }) + .messages([ + { role: 'user', content: 'Hello' + new Date().getTime() }, + ]).options; + + // Make the request + const response = await fetch(url, options); + const data: any = await response.json(); + + // The next request should be a hit + const response1 = await fetch(url, options); + const data1: any = await response1.json(); + + expect(response1.headers.get('x-portkey-cache-status')).toBe('HIT'); + + // Wait 2 seconds + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Make the request again + const response2 = await fetch(url, options); + const data2: any = await response2.json(); + + expect(response2.status).toBe(200); + expect(response2.headers.get('x-portkey-cache-status')).toBe('MISS'); + expect(data2.choices[0].message.content).toBeDefined(); + }); + + it.skip('should handle cache with streaming responses correctly', async () => { + // Verify streaming from cache works + const url = urlBuilder.chat(); + const options = requestBuilder + .config({ + cache: { mode: 'simple' }, + }) + .stream(true) + .messages([ + { role: 'user', content: 'Hello' + new Date().getTime() }, + ]).options; + + // Store in cache + const nonCachedResponse = await fetch(url, options); + // The response should be a stream + expect(nonCachedResponse.body).toBeInstanceOf(ReadableStream); + + expect(nonCachedResponse.status).toBe(200); + expect(nonCachedResponse.headers.get('x-portkey-cache-status')).toBe( + 'MISS' + ); + + // Get from cache + const response = await fetch(url, options); + // The response should be a stream + expect(response.body).toBeInstanceOf(ReadableStream); + + expect(response.status).toBe(200); + expect(response.headers.get('x-portkey-cache-status')).toBe('HIT'); + }); +}); diff --git a/tests/unit/src/handlers/services/benchmark.ts b/tests/unit/src/handlers/services/benchmark.ts new file mode 100644 index 0000000..341cda3 --- /dev/null +++ b/tests/unit/src/handlers/services/benchmark.ts @@ -0,0 +1,159 @@ +import { + LogsService, + LogObjectBuilder, +} from '../../../../../src/handlers/services/logsService.js'; +import type { Context } from 'hono'; +import type { RequestContext } from '../../../../../src/handlers/services/requestContext.js'; + +// Helper function to create sample data of different sizes +function createSampleData(size: number) { + const data: any = { + nested: {}, + array: [], + }; + + for (let i = 0; i < size; i++) { + data.nested[`key${i}`] = { + value: `value${i}`, + timestamp: new Date(), + metadata: { + id: i, + tags: ['tag1', 'tag2'], + }, + }; + data.array.push({ + index: i, + data: Buffer.from(`data${i}`).toString('base64'), + objects: Array(10) + .fill(null) + .map((_, j) => ({ subIndex: j })), + }); + } + + return data; +} + +// Mock Context and RequestContext for testing +const mockContext = { + get: () => [], + set: () => {}, +} as unknown as Context; + +const mockRequestContext = { + providerOption: { + someOption: 'value', + }, + requestURL: 'https://api.example.com', + endpoint: '/test', + requestBody: { test: 'body' }, + index: 0, + cacheConfig: { + mode: 'default', + maxAge: 3600, + }, + transformedRequestBody: { transformed: 'body' }, + params: { param: 'value' }, +} as unknown as RequestContext; + +function measureOperation(name: string, fn: () => void, indent: number = 0) { + const start = performance.now(); + fn(); + const duration = performance.now() - start; + console.log(`${' '.repeat(indent)}${name}: ${duration.toFixed(3)}ms`); + return duration; +} + +// Measure multiple iterations to see JIT optimization +function measureWithIterations( + name: string, + fn: () => void, + iterations: number = 10 +) { + const times: number[] = []; + console.log(`\n${name} (${iterations} iterations):`); + + for (let i = 0; i < iterations; i++) { + const start = performance.now(); + fn(); + times.push(performance.now() - start); + } + + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const min = Math.min(...times); + const max = Math.max(...times); + + console.log(` Average: ${avg.toFixed(3)}ms`); + console.log(` Min: ${min.toFixed(3)}ms`); + console.log(` Max: ${max.toFixed(3)}ms`); + console.log(` First: ${times[0].toFixed(3)}ms`); + console.log(` Last: ${times[times.length - 1].toFixed(3)}ms`); + + return avg; +} + +// Benchmark scenarios +async function runBenchmarks() { + console.log('Starting log operation analysis...\n'); + + const scenarios = [ + { name: 'Small payload', size: 10 }, + { name: 'Medium payload', size: 100 }, + { name: 'Large payload', size: 1000 }, + ]; + + for (const scenario of scenarios) { + console.log(`\nScenario: ${scenario.name}`); + console.log('='.repeat(20)); + const sampleData = createSampleData(scenario.size); + + // Initialize and setup + const logsService = new LogsService(mockContext); + const builder = new LogObjectBuilder(logsService, mockRequestContext); + + // Setup the builder with data + builder.updateRequestContext(mockRequestContext, { + 'Content-Type': 'application/json', + }); + builder.addTransformedRequest(sampleData, { 'X-Custom': 'value' }); + builder.addResponse( + new Response(JSON.stringify(sampleData), { + headers: { 'Content-Type': 'application/json' }, + }), + sampleData + ); + builder.addExecutionTime(new Date()); + builder.addCache('hit', 'test-key'); + builder.addHookSpanId('span-123'); + + // Measure individual operations with iterations + measureWithIterations('Validation check', () => { + (builder as any).isComplete((builder as any).logData); + }); + + measureWithIterations('Clone operation', () => { + (builder as any).clone(); + }); + + measureWithIterations('Full log operation', () => { + builder.log(); + }); + + // Memory usage + const memoryUsage = process.memoryUsage(); + console.log('\nMemory usage:'); + console.log( + ` - Heap used: ${(memoryUsage.heapUsed / 1024 / 1024).toFixed(2)} MB` + ); + console.log( + ` - Heap total: ${(memoryUsage.heapTotal / 1024 / 1024).toFixed(2)} MB` + ); + + builder.commit(); + } +} + +// Run benchmarks +console.log('Running log operation analysis...'); +runBenchmarks() + .then(() => console.log('\nAnalysis completed')) + .catch(console.error); diff --git a/tests/unit/src/handlers/services/cacheService.test.ts b/tests/unit/src/handlers/services/cacheService.test.ts new file mode 100644 index 0000000..b83b40c --- /dev/null +++ b/tests/unit/src/handlers/services/cacheService.test.ts @@ -0,0 +1,346 @@ +import { Context } from 'hono'; +import { CacheService } from '../../../../../src/handlers/services/cacheService'; +import { HooksService } from '../../../../../src/handlers/services/hooksService'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import { endpointStrings } from '../../../../../src/providers/types'; + +// Mock HooksService +jest.mock('../hooksService'); + +// Mock env function +jest.mock('hono/adapter', () => ({ + env: jest.fn(() => ({})), +})); + +describe('CacheService', () => { + let mockContext: Context; + let mockHooksService: jest.Mocked; + let mockRequestContext: RequestContext; + let cacheService: CacheService; + + beforeEach(() => { + mockContext = { + get: jest.fn().mockReturnValue(undefined), + } as unknown as Context; + + mockHooksService = { + results: { + beforeRequestHooksResult: [], + afterRequestHooksResult: [], + }, + hasFailedHooks: jest.fn(), + } as unknown as jest.Mocked; + + mockRequestContext = { + endpoint: 'chatComplete' as endpointStrings, + honoContext: mockContext, + requestHeaders: {}, + transformedRequestBody: { message: 'test' }, + cacheConfig: { + mode: 'simple', + maxAge: 3600, + }, + } as unknown as RequestContext; + + cacheService = new CacheService(mockContext, mockHooksService); + }); + + describe('isEndpointCacheable', () => { + it('should return true for cacheable endpoints', () => { + expect(cacheService.isEndpointCacheable('chatComplete')).toBe(true); + expect(cacheService.isEndpointCacheable('complete')).toBe(true); + expect(cacheService.isEndpointCacheable('embed')).toBe(true); + expect(cacheService.isEndpointCacheable('imageGenerate')).toBe(true); + }); + + it('should return false for non-cacheable endpoints', () => { + expect(cacheService.isEndpointCacheable('uploadFile')).toBe(false); + expect(cacheService.isEndpointCacheable('listFiles')).toBe(false); + expect(cacheService.isEndpointCacheable('retrieveFile')).toBe(false); + expect(cacheService.isEndpointCacheable('deleteFile')).toBe(false); + expect(cacheService.isEndpointCacheable('createBatch')).toBe(false); + expect(cacheService.isEndpointCacheable('retrieveBatch')).toBe(false); + expect(cacheService.isEndpointCacheable('cancelBatch')).toBe(false); + expect(cacheService.isEndpointCacheable('listBatches')).toBe(false); + expect(cacheService.isEndpointCacheable('getBatchOutput')).toBe(false); + expect(cacheService.isEndpointCacheable('listFinetunes')).toBe(false); + expect(cacheService.isEndpointCacheable('createFinetune')).toBe(false); + expect(cacheService.isEndpointCacheable('retrieveFinetune')).toBe(false); + expect(cacheService.isEndpointCacheable('cancelFinetune')).toBe(false); + }); + }); + + describe('getFromCacheFunction', () => { + it('should return cache function from context', () => { + const mockCacheFunction = jest.fn(); + (mockContext.get as jest.Mock).mockReturnValue(mockCacheFunction); + + expect(cacheService.getFromCacheFunction).toBe(mockCacheFunction); + expect(mockContext.get).toHaveBeenCalledWith('getFromCache'); + }); + + it('should return undefined if no cache function', () => { + (mockContext.get as jest.Mock).mockReturnValue(undefined); + + expect(cacheService.getFromCacheFunction).toBeUndefined(); + }); + }); + + describe('getCacheIdentifier', () => { + it('should return cache identifier from context', () => { + const mockIdentifier = 'cache-id-123'; + (mockContext.get as jest.Mock).mockReturnValue(mockIdentifier); + + expect(cacheService.getCacheIdentifier).toBe(mockIdentifier); + expect(mockContext.get).toHaveBeenCalledWith('cacheIdentifier'); + }); + }); + + describe('noCacheObject', () => { + it('should return default no-cache object', () => { + const result = cacheService.noCacheObject; + + expect(result).toEqual({ + cacheResponse: undefined, + cacheStatus: 'DISABLED', + cacheKey: undefined, + createdAt: expect.any(Date), + }); + }); + }); + + describe('getCachedResponse', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should return no-cache object for non-cacheable endpoints', async () => { + const context = { + ...mockRequestContext, + endpoint: 'uploadFile' as endpointStrings, + } as RequestContext; + + const result = await cacheService.getCachedResponse(context, {}); + + expect(result).toEqual({ + cacheResponse: undefined, + cacheStatus: 'DISABLED', + cacheKey: undefined, + createdAt: expect.any(Date), + }); + }); + + it('should return no-cache object when cache function is not available', async () => { + (mockContext.get as jest.Mock).mockReturnValue(undefined); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result).toEqual({ + cacheResponse: undefined, + cacheStatus: 'DISABLED', + cacheKey: undefined, + createdAt: expect.any(Date), + }); + }); + + it('should return no-cache object when cache mode is not set', async () => { + const context = { + ...mockRequestContext, + cacheConfig: { mode: undefined, maxAge: undefined }, + } as unknown as RequestContext; + + const result = await cacheService.getCachedResponse(context, {}); + + expect(result).toEqual({ + cacheResponse: undefined, + cacheStatus: 'DISABLED', + cacheKey: undefined, + createdAt: expect.any(Date), + }); + }); + + it('should return cache response when cache hit', async () => { + const mockCacheFunction = jest + .fn() + .mockResolvedValue([ + '{"choices": [{"message": {"content": "cached response"}}]}', + 'HIT', + 'cache-key-123', + ]); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheResponse).toBeInstanceOf(Response); + expect(result.cacheStatus).toBe('HIT'); + expect(result.cacheKey).toBe('cache-key-123'); + expect(result.createdAt).toBeInstanceOf(Date); + }); + + it('should return cache miss when no cached response', async () => { + const mockCacheFunction = jest + .fn() + .mockResolvedValue([null, 'MISS', 'cache-key-123']); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheResponse).toBeUndefined(); + expect(result.cacheStatus).toBe('MISS'); + expect(result.cacheKey).toBe('cache-key-123'); + }); + + it('should include hook results in cached response when available', async () => { + const mockHookResults = [ + { id: 'hook1', verdict: true }, + { id: 'hook2', verdict: false }, + ]; + Object.defineProperty(mockHooksService, 'results', { + value: { + beforeRequestHooksResult: mockHookResults, + afterRequestHooksResult: [], + }, + writable: true, + }); + mockHooksService.hasFailedHooks.mockReturnValue(true); + + const mockCacheFunction = jest + .fn() + .mockResolvedValue([ + '{"choices": [{"message": {"content": "cached response"}}]}', + 'HIT', + 'cache-key-123', + ]); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheResponse).toBeInstanceOf(Response); + expect(result.cacheResponse!.status).toBe(246); // Failed hooks status + + const responseBody = await result.cacheResponse!.json(); + expect(responseBody).toHaveProperty('hook_results'); + expect((responseBody as any).hook_results.before_request_hooks).toEqual( + mockHookResults + ); + }); + + it('should return status 200 when no failed hooks', async () => { + const mockHookResults = [ + { id: 'hook1', verdict: true }, + { id: 'hook2', verdict: true }, + ]; + Object.defineProperty(mockHooksService, 'results', { + value: { + beforeRequestHooksResult: mockHookResults, + afterRequestHooksResult: [], + }, + writable: true, + }); + mockHooksService.hasFailedHooks.mockReturnValue(false); + + const mockCacheFunction = jest + .fn() + .mockResolvedValue([ + '{"choices": [{"message": {"content": "cached response"}}]}', + 'HIT', + 'cache-key-123', + ]); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheResponse!.status).toBe(200); + }); + + it('should handle cache function parameters correctly', async () => { + const mockCacheFunction = jest + .fn() + .mockResolvedValue([null, 'MISS', null]); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const headers = { authorization: 'Bearer test' }; + await cacheService.getCachedResponse(mockRequestContext, headers); + + expect(mockCacheFunction).toHaveBeenCalledWith( + {}, // env result + { ...mockRequestContext.requestHeaders, ...headers }, + mockRequestContext.transformedRequestBody, + mockRequestContext.endpoint, + 'cache-identifier', + 'simple', + 3600 + ); + }); + + it('should handle undefined cache status and key', async () => { + const mockCacheFunction = jest + .fn() + .mockResolvedValue([null, undefined, undefined]); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheStatus).toBe('DISABLED'); + expect(result.cacheKey).toBeUndefined(); + }); + + it('should handle empty cache key', async () => { + const mockCacheFunction = jest.fn().mockResolvedValue([null, 'MISS', '']); + (mockContext.get as jest.Mock).mockImplementation((key: string) => { + if (key === 'getFromCache') return mockCacheFunction; + if (key === 'cacheIdentifier') return 'cache-identifier'; + return undefined; + }); + + const result = await cacheService.getCachedResponse( + mockRequestContext, + {} + ); + + expect(result.cacheKey).toBeUndefined(); + }); + }); +}); diff --git a/tests/unit/src/handlers/services/hooksService.test.ts b/tests/unit/src/handlers/services/hooksService.test.ts new file mode 100644 index 0000000..46ace8f --- /dev/null +++ b/tests/unit/src/handlers/services/hooksService.test.ts @@ -0,0 +1,306 @@ +import { HooksService } from '../../../../../src/handlers/services/hooksService'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import { HooksManager, HookSpan } from '../../../../../src/middlewares/hooks'; +import { + HookType, + AllHookResults, + GuardrailResult, + HookObject, +} from '../../../../../src/middlewares/hooks/types'; + +// Mock the HooksManager and HookSpan +jest.mock('../../../middlewares/hooks'); + +describe('HooksService', () => { + let mockRequestContext: RequestContext; + let mockHooksManager: jest.Mocked; + let mockHookSpan: jest.Mocked; + let hooksService: HooksService; + + beforeEach(() => { + mockHookSpan = { + id: 'span-123', + getHooksResult: jest.fn(), + } as unknown as jest.Mocked; + + mockHooksManager = { + createSpan: jest.fn().mockReturnValue(mockHookSpan), + getHooksToExecute: jest.fn(), + } as unknown as jest.Mocked; + + mockRequestContext = { + params: { message: 'test' }, + metadata: { userId: '123' }, + provider: 'openai', + isStreaming: false, + beforeRequestHooks: [], + afterRequestHooks: [], + endpoint: 'chatComplete', + requestHeaders: {}, + hooksManager: mockHooksManager, + } as unknown as RequestContext; + + hooksService = new HooksService(mockRequestContext); + }); + + describe('constructor', () => { + it('should create hooks service and initialize span', () => { + expect(mockHooksManager.createSpan).toHaveBeenCalledWith( + mockRequestContext.params, + mockRequestContext.metadata, + mockRequestContext.provider, + mockRequestContext.isStreaming, + mockRequestContext.beforeRequestHooks, + mockRequestContext.afterRequestHooks, + null, + mockRequestContext.endpoint, + mockRequestContext.requestHeaders + ); + }); + }); + + describe('createSpan', () => { + it('should create and return a new hook span', () => { + const newMockSpan = { id: 'new-span-456' } as HookSpan; + mockHooksManager.createSpan.mockReturnValue(newMockSpan); + + const result = hooksService.createSpan(); + + expect(result).toBe(newMockSpan); + expect(mockHooksManager.createSpan).toHaveBeenCalledTimes(2); // Once in constructor, once here + }); + }); + + describe('hookSpan getter', () => { + it('should return the current hook span', () => { + expect(hooksService.hookSpan).toBe(mockHookSpan); + }); + }); + + describe('results getter', () => { + it('should return hook results from span', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'hook1', verdict: true } as GuardrailResult, + { id: 'hook2', verdict: false } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'hook3', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.results).toBe(mockResults); + expect(mockHookSpan.getHooksResult).toHaveBeenCalled(); + }); + + it('should return undefined when no results', () => { + mockHookSpan.getHooksResult.mockReturnValue(undefined as any); + + expect(hooksService.results).toBeUndefined(); + }); + }); + + describe('areSyncHooksAvailable getter', () => { + it('should return true when sync hooks are available', () => { + mockHooksManager.getHooksToExecute.mockReturnValue([ + { + id: 'hook1', + type: HookType.GUARDRAIL, + eventType: 'beforeRequestHook', + } as HookObject, + { + id: 'hook2', + type: HookType.MUTATOR, + eventType: 'afterRequestHook', + } as HookObject, + ]); + + expect(hooksService.areSyncHooksAvailable).toBe(true); + expect(mockHooksManager.getHooksToExecute).toHaveBeenCalledWith( + mockHookSpan, + ['syncBeforeRequestHook', 'syncAfterRequestHook'] + ); + }); + + it('should return false when no sync hooks available', () => { + mockHooksManager.getHooksToExecute.mockReturnValue([]); + + expect(hooksService.areSyncHooksAvailable).toBe(false); + }); + + it('should return false when hook span is not available', () => { + hooksService = new HooksService({ + ...mockRequestContext, + hooksManager: { + ...mockHooksManager, + createSpan: jest.fn().mockReturnValue(null), + }, + } as unknown as RequestContext); + + expect(hooksService.areSyncHooksAvailable).toBe(false); + }); + }); + + describe('hasFailedHooks', () => { + beforeEach(() => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: true } as GuardrailResult, + { id: 'brh2', verdict: false } as GuardrailResult, + { id: 'brh3', verdict: true } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'arh1', verdict: false } as GuardrailResult, + { id: 'arh2', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + }); + + it('should return true for beforeRequest when there are failed before request hooks', () => { + expect(hooksService.hasFailedHooks('beforeRequest')).toBe(true); + }); + + it('should return true for afterRequest when there are failed after request hooks', () => { + expect(hooksService.hasFailedHooks('afterRequest')).toBe(true); + }); + + it('should return true for any when there are failed hooks in either category', () => { + expect(hooksService.hasFailedHooks('any')).toBe(true); + }); + + it('should return false for beforeRequest when all before request hooks pass', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: true } as GuardrailResult, + { id: 'brh2', verdict: true } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'arh1', verdict: false } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); + }); + + it('should return false for afterRequest when all after request hooks pass', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: false } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'arh1', verdict: true } as GuardrailResult, + { id: 'arh2', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); + }); + + it('should return false for any when all hooks pass', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: true } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'arh1', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasFailedHooks('any')).toBe(false); + }); + + it('should handle empty hook results', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [], + afterRequestHooksResult: [], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); + expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); + expect(hooksService.hasFailedHooks('any')).toBe(false); + }); + + it('should handle undefined hook results', () => { + mockHookSpan.getHooksResult.mockReturnValue(undefined as any); + + expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); + expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); + expect(hooksService.hasFailedHooks('any')).toBe(false); + }); + }); + + describe('hasResults', () => { + beforeEach(() => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: true } as GuardrailResult, + { id: 'brh2', verdict: false } as GuardrailResult, + ], + afterRequestHooksResult: [ + { id: 'arh1', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + }); + + it('should return true for beforeRequest when there are before request hook results', () => { + expect(hooksService.hasResults('beforeRequest')).toBe(true); + }); + + it('should return true for afterRequest when there are after request hook results', () => { + expect(hooksService.hasResults('afterRequest')).toBe(true); + }); + + it('should return true for any when there are results in either category', () => { + expect(hooksService.hasResults('any')).toBe(true); + }); + + it('should return false for beforeRequest when no before request hook results', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [], + afterRequestHooksResult: [ + { id: 'arh1', verdict: true } as GuardrailResult, + ], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasResults('beforeRequest')).toBe(false); + }); + + it('should return false for afterRequest when no after request hook results', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [ + { id: 'brh1', verdict: true } as GuardrailResult, + ], + afterRequestHooksResult: [], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasResults('afterRequest')).toBe(false); + }); + + it('should return false for any when no results in either category', () => { + const mockResults: AllHookResults = { + beforeRequestHooksResult: [], + afterRequestHooksResult: [], + }; + mockHookSpan.getHooksResult.mockReturnValue(mockResults); + + expect(hooksService.hasResults('any')).toBe(false); + }); + + it('should handle undefined hook results', () => { + mockHookSpan.getHooksResult.mockReturnValue(undefined as any); + + expect(hooksService.hasResults('beforeRequest')).toBe(false); + expect(hooksService.hasResults('afterRequest')).toBe(false); + expect(hooksService.hasResults('any')).toBe(false); + }); + }); +}); diff --git a/tests/unit/src/handlers/services/logsService.test.ts b/tests/unit/src/handlers/services/logsService.test.ts new file mode 100644 index 0000000..ffdbc73 --- /dev/null +++ b/tests/unit/src/handlers/services/logsService.test.ts @@ -0,0 +1,625 @@ +import { Context } from 'hono'; +import { + LogsService, + LogObjectBuilder, +} from '../../../../../src/handlers/services/logsService'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; +import { ToolCall } from '../../../../../src/types/requestBody'; + +describe('LogsService', () => { + let mockContext: Context; + let logsService: LogsService; + + beforeEach(() => { + mockContext = { + get: jest.fn(), + set: jest.fn(), + } as unknown as Context; + + logsService = new LogsService(mockContext); + }); + + // Mock crypto for Node.js environment + const mockCrypto = { + randomUUID: jest.fn(() => 'mock-uuid-123'), + }; + (global as any).crypto = mockCrypto; + + describe('createExecuteToolSpan', () => { + const mockToolCall: ToolCall = { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + description: 'Get current weather', + arguments: '{"location": "New York"}', + }, + }; + + const mockToolOutput = { + temperature: '20°C', + condition: 'sunny', + }; + + it('should create execute tool span with correct structure', () => { + const startTime = 1000000000; + const endTime = 1000001000; + const traceId = 'trace-123'; + const parentSpanId = 'parent-456'; + const spanId = 'span-789'; + + const result = logsService.createExecuteToolSpan( + mockToolCall, + mockToolOutput, + startTime, + endTime, + traceId, + parentSpanId, + spanId + ); + + expect(result).toEqual({ + type: 'otlp_span', + traceId: 'trace-123', + spanId: 'span-789', + parentSpanId: 'parent-456', + name: 'execute_tool get_weather', + kind: 'SPAN_KIND_INTERNAL', + startTimeUnixNano: startTime, + endTimeUnixNano: endTime, + status: { + code: 'STATUS_CODE_OK', + }, + attributes: [ + { + key: 'gen_ai.operation.name', + value: { stringValue: 'execute_tool' }, + }, + { + key: 'gen_ai.tool.name', + value: { stringValue: 'get_weather' }, + }, + { + key: 'gen_ai.tool.description', + value: { stringValue: 'Get current weather' }, + }, + ], + events: [ + { + timeUnixNano: startTime, + name: 'gen_ai.tool.input', + attributes: [ + { + key: 'location', + value: { stringValue: 'New York' }, + }, + ], + }, + { + timeUnixNano: endTime, + name: 'gen_ai.tool.output', + attributes: [ + { + key: 'temperature', + value: { stringValue: '20°C' }, + }, + { + key: 'condition', + value: { stringValue: 'sunny' }, + }, + ], + }, + ], + }); + }); + + it('should generate random span ID when not provided', () => { + const result = logsService.createExecuteToolSpan( + mockToolCall, + mockToolOutput, + 1000, + 2000, + 'trace-123' + ); + + expect(result.spanId).toBe('mock-uuid-123'); + expect(mockCrypto.randomUUID).toHaveBeenCalled(); + }); + + it('should handle undefined parent span ID', () => { + const result = logsService.createExecuteToolSpan( + mockToolCall, + mockToolOutput, + 1000, + 2000, + 'trace-123' + ); + + expect(result.parentSpanId).toBeUndefined(); + }); + }); + + describe('createLogObject', () => { + let mockRequestContext: RequestContext; + let mockProviderContext: ProviderContext; + let mockResponse: Response; + + beforeEach(() => { + mockRequestContext = { + providerOption: { provider: 'openai' }, + requestURL: 'https://api.openai.com/v1/chat/completions', + endpoint: 'chatComplete', + transformedRequestBody: { model: 'gpt-4', messages: [] }, + params: { model: 'gpt-4', messages: [] }, + index: 0, + cacheConfig: { mode: 'simple', maxAge: 3600 }, + } as unknown as RequestContext; + + mockProviderContext = {} as ProviderContext; + + mockResponse = new Response('{"choices": []}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + + it('should create log object with all required fields', async () => { + const hookSpanId = 'hook-span-123'; + const cacheKey = 'cache-key-456'; + const fetchOptions = { + headers: { authorization: 'Bearer sk-test' }, + }; + const cacheStatus = 'MISS'; + const originalResponseJSON = { choices: [] }; + const createdAt = new Date('2024-01-01T00:00:00Z'); + const executionTime = 1500; + + const result = await logsService.createLogObject( + mockRequestContext, + mockProviderContext, + hookSpanId, + cacheKey, + fetchOptions, + cacheStatus, + mockResponse, + originalResponseJSON, + createdAt, + executionTime + ); + + expect(result).toEqual({ + providerOptions: { + provider: 'openai', + requestURL: 'https://api.openai.com/v1/chat/completions', + rubeusURL: 'chatComplete', + }, + transformedRequest: { + body: { model: 'gpt-4', messages: [] }, + headers: { authorization: 'Bearer sk-test' }, + }, + requestParams: { model: 'gpt-4', messages: [] }, + finalUntransformedRequest: { + body: { model: 'gpt-4', messages: [] }, + }, + originalResponse: { + body: { choices: [] }, + }, + createdAt, + response: expect.any(Response), + cacheStatus: 'MISS', + lastUsedOptionIndex: 0, + cacheKey: 'cache-key-456', + cacheMode: 'simple', + cacheMaxAge: 3600, + hookSpanId: 'hook-span-123', + executionTime: 1500, + }); + }); + + it('should use current date when createdAt not provided', async () => { + const beforeCall = new Date(); + + const result = await logsService.createLogObject( + mockRequestContext, + mockProviderContext, + 'hook-span-123', + undefined, + {}, + undefined, + mockResponse, + null + ); + + const afterCall = new Date(); + expect(result.createdAt.getTime()).toBeGreaterThanOrEqual( + beforeCall.getTime() + ); + expect(result.createdAt.getTime()).toBeLessThanOrEqual( + afterCall.getTime() + ); + }); + + it('should handle undefined optional parameters', async () => { + const result = await logsService.createLogObject( + mockRequestContext, + mockProviderContext, + 'hook-span-123', + undefined, + {}, + undefined, + mockResponse, + undefined + ); + + expect(result.cacheKey).toBeUndefined(); + expect(result.cacheStatus).toBeUndefined(); + expect(result.originalResponse.body).toBeUndefined(); + expect(result.executionTime).toBeUndefined(); + }); + }); + + describe('requestLogs getter', () => { + it('should return logs from context', () => { + const mockLogs = [{ id: 'log1' }, { id: 'log2' }]; + (mockContext.get as jest.Mock).mockReturnValue(mockLogs); + + expect(logsService.requestLogs).toBe(mockLogs); + expect(mockContext.get).toHaveBeenCalledWith('requestOptions'); + }); + + it('should return empty array when no logs in context', () => { + (mockContext.get as jest.Mock).mockReturnValue(undefined); + + expect(logsService.requestLogs).toEqual([]); + }); + }); + + describe('addRequestLog', () => { + it('should add log to existing logs', () => { + const existingLogs = [{ id: 'log1' }]; + const newLog = { id: 'log2' }; + (mockContext.get as jest.Mock).mockReturnValue(existingLogs); + + logsService.addRequestLog(newLog); + + expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ + { id: 'log1' }, + { id: 'log2' }, + ]); + }); + + it('should add log when no existing logs', () => { + const newLog = { id: 'log1' }; + (mockContext.get as jest.Mock).mockReturnValue([]); + + logsService.addRequestLog(newLog); + + expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ + { id: 'log1' }, + ]); + }); + }); +}); + +describe('LogObjectBuilder', () => { + let mockLogsService: LogsService; + let mockRequestContext: RequestContext; + let logObjectBuilder: LogObjectBuilder; + + beforeEach(() => { + mockLogsService = { + addRequestLog: jest.fn(), + } as unknown as LogsService; + + mockRequestContext = { + providerOption: { provider: 'openai' }, + requestURL: 'https://api.openai.com/v1/chat/completions', + endpoint: 'chatComplete', + requestBody: { model: 'gpt-4', messages: [] }, + index: 0, + cacheConfig: { mode: 'simple', maxAge: 3600 }, + } as unknown as RequestContext; + + logObjectBuilder = new LogObjectBuilder( + mockLogsService, + mockRequestContext + ); + }); + + describe('constructor', () => { + it('should initialize log data with request context', () => { + const builder = new LogObjectBuilder(mockLogsService, mockRequestContext); + + // Test by calling log and checking the data passed to addRequestLog + builder.log(); + + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + providerOptions: { + provider: 'openai', + requestURL: 'https://api.openai.com/v1/chat/completions', + rubeusURL: 'chatComplete', + }, + finalUntransformedRequest: { + body: { model: 'gpt-4', messages: [] }, + }, + lastUsedOptionIndex: 0, + cacheMode: 'simple', + cacheMaxAge: 3600, + createdAt: expect.any(Date), + }) + ); + }); + }); + + describe('updateRequestContext', () => { + it('should update request context data', () => { + const updatedContext = { + ...mockRequestContext, + index: 1, + transformedRequestBody: { model: 'gpt-3.5-turbo', messages: [] }, + params: { model: 'gpt-3.5-turbo', messages: [] }, + } as unknown as RequestContext; + const headers = { authorization: 'Bearer sk-test' }; + + const result = logObjectBuilder.updateRequestContext( + updatedContext, + headers + ); + + expect(result).toBe(logObjectBuilder); + + // Verify data was updated by calling log + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + lastUsedOptionIndex: 1, + transformedRequest: { + body: { model: 'gpt-3.5-turbo', messages: [] }, + headers: { authorization: 'Bearer sk-test' }, + }, + requestParams: { model: 'gpt-3.5-turbo', messages: [] }, + }) + ); + }); + + it('should handle undefined headers', () => { + const result = logObjectBuilder.updateRequestContext(mockRequestContext); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + transformedRequest: expect.objectContaining({ + headers: {}, + }), + }) + ); + }); + }); + + describe('addResponse', () => { + it('should add response data', () => { + const mockResponse = new Response('{"test": true}'); + const originalJson = { test: true }; + + const result = logObjectBuilder.addResponse(mockResponse, originalJson); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + response: expect.any(Response), + originalResponse: { + body: { test: true }, + }, + }) + ); + }); + + it('should handle null original response JSON', () => { + const mockResponse = new Response('{}'); + + logObjectBuilder.addResponse(mockResponse, null); + logObjectBuilder.log(); + + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + originalResponse: { + body: null, + }, + }) + ); + }); + }); + + describe('addExecutionTime', () => { + it('should set creation time and calculate execution time', () => { + const createdAt = new Date('2024-01-01T00:00:00Z'); + const currentTime = Date.now(); + const originalDateNow = Date.now; + Date.now = jest.fn(() => currentTime); + + const result = logObjectBuilder.addExecutionTime(createdAt); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + createdAt, + executionTime: currentTime - createdAt.getTime(), + }) + ); + + Date.now = originalDateNow; + }); + }); + + describe('addTransformedRequest', () => { + it('should add transformed request data', () => { + const transformedBody = { model: 'claude-3', messages: [] }; + const transformedHeaders = { 'x-api-key': 'sk-ant-test' }; + + const result = logObjectBuilder.addTransformedRequest( + transformedBody, + transformedHeaders + ); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + transformedRequest: { + body: transformedBody, + headers: transformedHeaders, + }, + }) + ); + }); + }); + + describe('addCache', () => { + it('should add cache data', () => { + const result = logObjectBuilder.addCache('HIT', 'cache-key-123'); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + cacheStatus: 'HIT', + cacheKey: 'cache-key-123', + }) + ); + }); + + it('should handle undefined cache parameters', () => { + const result = logObjectBuilder.addCache(); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + cacheStatus: undefined, + cacheKey: undefined, + }) + ); + }); + }); + + describe('addHookSpanId', () => { + it('should add hook span ID', () => { + const result = logObjectBuilder.addHookSpanId('hook-span-789'); + + expect(result).toBe(logObjectBuilder); + + logObjectBuilder.log(); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + hookSpanId: 'hook-span-789', + }) + ); + }); + }); + + describe('log', () => { + it('should call logsService.addRequestLog with log data', () => { + // Set up minimum required data to pass validation + logObjectBuilder + .addTransformedRequest({}, {}) + .addResponse(new Response('{}'), {}) + .addHookSpanId('test-span-id'); + + logObjectBuilder.log(); + + expect(mockLogsService.addRequestLog).toHaveBeenCalledTimes(1); + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + providerOptions: expect.any(Object), + finalUntransformedRequest: expect.any(Object), + createdAt: expect.any(Date), + }) + ); + }); + + it('should calculate execution time when createdAt is set', () => { + const createdAt = new Date(Date.now() - 1000); // 1 second ago + logObjectBuilder + .addTransformedRequest({}, {}) + .addResponse(new Response('{}'), {}) + .addHookSpanId('test-span-id') + .addExecutionTime(createdAt); + + logObjectBuilder.log(); + + expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( + expect.objectContaining({ + executionTime: expect.any(Number), + }) + ); + }); + + it('should throw error when trying to log from committed object', () => { + logObjectBuilder.commit(); + + expect(() => logObjectBuilder.log()).toThrow( + 'Cannot log from a committed log object' + ); + }); + + it('should return self for method chaining', () => { + logObjectBuilder + .addTransformedRequest({}, {}) + .addResponse(new Response('{}'), {}) + .addHookSpanId('test-span-id'); + + const result = logObjectBuilder.log(); + expect(result).toBe(logObjectBuilder); + }); + }); + + describe('commit', () => { + it('should mark object as committed', () => { + expect(logObjectBuilder.isDestroyed()).toBe(false); + + logObjectBuilder.commit(); + + expect(logObjectBuilder.isDestroyed()).toBe(true); + }); + + it('should be safe to call multiple times', () => { + logObjectBuilder.commit(); + logObjectBuilder.commit(); // Should not throw + + expect(logObjectBuilder.isDestroyed()).toBe(true); + }); + }); + + describe('isDestroyed', () => { + it('should return false for new object', () => { + expect(logObjectBuilder.isDestroyed()).toBe(false); + }); + + it('should return true after commit', () => { + logObjectBuilder.commit(); + expect(logObjectBuilder.isDestroyed()).toBe(true); + }); + }); + + describe('Symbol.dispose', () => { + it('should call commit when disposed', () => { + const commitSpy = jest.spyOn(logObjectBuilder, 'commit'); + + logObjectBuilder[Symbol.dispose](); + + expect(commitSpy).toHaveBeenCalled(); + expect(logObjectBuilder.isDestroyed()).toBe(true); + }); + }); +}); diff --git a/tests/unit/src/handlers/services/preRequestValidatorService.test.ts b/tests/unit/src/handlers/services/preRequestValidatorService.test.ts new file mode 100644 index 0000000..0a148c5 --- /dev/null +++ b/tests/unit/src/handlers/services/preRequestValidatorService.test.ts @@ -0,0 +1,230 @@ +import { Context } from 'hono'; +import { PreRequestValidatorService } from '../../../../../src/handlers/services/preRequestValidatorService'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; + +describe('PreRequestValidatorService', () => { + let mockContext: Context; + let mockRequestContext: RequestContext; + let preRequestValidatorService: PreRequestValidatorService; + + beforeEach(() => { + mockContext = { + get: jest.fn(), + } as unknown as Context; + + mockRequestContext = { + providerOption: { provider: 'openai' }, + requestHeaders: { authorization: 'Bearer sk-test' }, + params: { model: 'gpt-4', messages: [] }, + } as unknown as RequestContext; + }); + + describe('constructor', () => { + it('should initialize with context and request context', () => { + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + expect(preRequestValidatorService).toBeInstanceOf( + PreRequestValidatorService + ); + expect(mockContext.get).toHaveBeenCalledWith('preRequestValidator'); + }); + }); + + describe('getResponse', () => { + it('should return undefined when no validator is set', async () => { + (mockContext.get as jest.Mock).mockReturnValue(undefined); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(result).toBeUndefined(); + expect(mockContext.get).toHaveBeenCalledWith('preRequestValidator'); + }); + + it('should call validator with correct parameters when validator exists', async () => { + const mockValidator = jest + .fn() + .mockResolvedValue( + new Response('{"error": "Validation failed"}', { status: 400 }) + ); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(mockValidator).toHaveBeenCalledWith( + mockContext, + mockRequestContext.providerOption, + mockRequestContext.requestHeaders, + mockRequestContext.params + ); + expect(result).toBeInstanceOf(Response); + expect(result!.status).toBe(400); + }); + + it('should return validator response when validation fails', async () => { + const errorResponse = new Response( + JSON.stringify({ + error: { + message: 'Budget exceeded', + type: 'budget_exceeded', + }, + }), + { + status: 429, + headers: { 'content-type': 'application/json' }, + } + ); + const mockValidator = jest.fn().mockResolvedValue(errorResponse); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(result).toBe(errorResponse); + expect(result!.status).toBe(429); + }); + + it('should return undefined when validator passes (returns null)', async () => { + const mockValidator = jest.fn().mockResolvedValue(null); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(mockValidator).toHaveBeenCalled(); + expect(result).toBeNull(); + }); + + it('should return undefined when validator passes (returns undefined)', async () => { + const mockValidator = jest.fn().mockResolvedValue(undefined); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(mockValidator).toHaveBeenCalled(); + expect(result).toBeUndefined(); + }); + + it('should handle validator that throws an error', async () => { + const mockValidator = jest + .fn() + .mockRejectedValue(new Error('Validator error')); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + await expect(preRequestValidatorService.getResponse()).rejects.toThrow( + 'Validator error' + ); + expect(mockValidator).toHaveBeenCalledWith( + mockContext, + mockRequestContext.providerOption, + mockRequestContext.requestHeaders, + mockRequestContext.params + ); + }); + + it('should handle async validator correctly', async () => { + const delayedResponse = new Promise((resolve) => { + setTimeout(() => { + resolve(new Response('{"status": "validated"}', { status: 200 })); + }, 10); + }); + const mockValidator = jest.fn().mockReturnValue(delayedResponse); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + preRequestValidatorService = new PreRequestValidatorService( + mockContext, + mockRequestContext + ); + + const result = await preRequestValidatorService.getResponse(); + + expect(result).toBeInstanceOf(Response); + expect(result!.status).toBe(200); + }); + + it('should pass correct parameters for different request contexts', async () => { + const customRequestContext = { + providerOption: { + provider: 'anthropic', + apiKey: 'sk-ant-test', + customParam: 'value', + }, + requestHeaders: { + authorization: 'Bearer sk-ant-test', + 'anthropic-version': '2023-06-01', + }, + params: { + model: 'claude-3-sonnet', + max_tokens: 1000, + messages: [{ role: 'user', content: 'Hello' }], + }, + } as unknown as RequestContext; + + const mockValidator = jest.fn().mockResolvedValue(undefined); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + const customService = new PreRequestValidatorService( + mockContext, + customRequestContext + ); + + await customService.getResponse(); + + expect(mockValidator).toHaveBeenCalledWith( + mockContext, + customRequestContext.providerOption, + customRequestContext.requestHeaders, + customRequestContext.params + ); + }); + + it('should handle empty request parameters', async () => { + const emptyRequestContext = { + providerOption: {}, + requestHeaders: {}, + params: {}, + } as unknown as RequestContext; + + const mockValidator = jest.fn().mockResolvedValue(undefined); + (mockContext.get as jest.Mock).mockReturnValue(mockValidator); + + const emptyService = new PreRequestValidatorService( + mockContext, + emptyRequestContext + ); + + await emptyService.getResponse(); + + expect(mockValidator).toHaveBeenCalledWith(mockContext, {}, {}, {}); + }); + }); +}); diff --git a/tests/unit/src/handlers/services/providerContext.test.ts b/tests/unit/src/handlers/services/providerContext.test.ts new file mode 100644 index 0000000..ea0ca9b --- /dev/null +++ b/tests/unit/src/handlers/services/providerContext.test.ts @@ -0,0 +1,465 @@ +import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import Providers from '../../../../../src/providers'; +import { ANTHROPIC, AZURE_OPEN_AI } from '../../../../../src/globals'; + +// Mock the Providers object +jest.mock('../../../providers', () => ({ + openai: { + api: { + headers: jest.fn(), + getBaseURL: jest.fn(), + getEndpoint: jest.fn(), + getProxyEndpoint: jest.fn(), + }, + requestHandlers: { + uploadFile: jest.fn(), + listFiles: jest.fn(), + }, + }, + anthropic: { + api: { + headers: jest.fn(), + getBaseURL: jest.fn(), + getEndpoint: jest.fn(), + }, + requestHandlers: {}, + }, + 'azure-openai': { + api: { + headers: jest.fn(), + getBaseURL: jest.fn(), + getEndpoint: jest.fn(), + }, + }, +})); + +describe('ProviderContext', () => { + let mockRequestContext: RequestContext; + + beforeEach(() => { + // Clean up any previous mocks + if (Providers.openai.api.getProxyEndpoint) { + delete Providers.openai.api.getProxyEndpoint; + } + + mockRequestContext = { + honoContext: { + req: { url: 'https://gateway.example.com/v1/chat/completions' }, + }, + providerOption: { provider: 'openai', apiKey: 'sk-test' }, + endpoint: 'chatComplete', + transformedRequestBody: { model: 'gpt-4', messages: [] }, + params: { model: 'gpt-4', messages: [] }, + } as unknown as RequestContext; + }); + + describe('constructor', () => { + it('should create provider context for valid provider', () => { + const context = new ProviderContext('openai'); + expect(context).toBeInstanceOf(ProviderContext); + }); + + it('should throw error for invalid provider', () => { + expect(() => new ProviderContext('invalid-provider')).toThrow( + 'Provider invalid-provider not found' + ); + }); + }); + + describe('providerConfig getter', () => { + it('should return provider config', () => { + const context = new ProviderContext('openai'); + expect(context.providerConfig).toBe(Providers.openai); + }); + }); + + describe('apiConfig getter', () => { + it('should return API config', () => { + const context = new ProviderContext('openai'); + expect(context.apiConfig).toBe(Providers.openai.api); + }); + }); + + describe('getHeaders', () => { + it('should call provider headers function with correct parameters', async () => { + const mockHeaders = { authorization: 'Bearer sk-test' }; + const mockHeadersFn = jest.fn().mockResolvedValue(mockHeaders); + Providers.openai.api.headers = mockHeadersFn; + + const context = new ProviderContext('openai'); + const result = await context.getHeaders(mockRequestContext); + + expect(mockHeadersFn).toHaveBeenCalledWith({ + c: mockRequestContext.honoContext, + providerOptions: mockRequestContext.providerOption, + fn: mockRequestContext.endpoint, + transformedRequestBody: mockRequestContext.transformedRequestBody, + transformedRequestUrl: mockRequestContext.honoContext.req.url, + gatewayRequestBody: mockRequestContext.params, + }); + expect(result).toBe(mockHeaders); + }); + + it('should handle async header generation', async () => { + const mockHeaders = { 'x-api-key': 'test-key' }; + const mockHeadersFn = jest.fn().mockResolvedValue(mockHeaders); + Providers.openai.api.headers = mockHeadersFn; + + const context = new ProviderContext('openai'); + const result = await context.getHeaders(mockRequestContext); + + expect(result).toEqual(mockHeaders); + }); + }); + + describe('getBaseURL', () => { + it('should call provider getBaseURL function with correct parameters', async () => { + const mockBaseURL = 'https://api.openai.com'; + const mockGetBaseURL = jest.fn().mockResolvedValue(mockBaseURL); + Providers.openai.api.getBaseURL = mockGetBaseURL; + + const context = new ProviderContext('openai'); + const result = await context.getBaseURL(mockRequestContext); + + expect(mockGetBaseURL).toHaveBeenCalledWith({ + providerOptions: mockRequestContext.providerOption, + fn: mockRequestContext.endpoint, + c: mockRequestContext.honoContext, + gatewayRequestURL: mockRequestContext.honoContext.req.url, + }); + expect(result).toBe(mockBaseURL); + }); + + it('should handle custom base URLs', async () => { + const customURL = 'https://custom.openai.com'; + const mockGetBaseURL = jest.fn().mockResolvedValue(customURL); + Providers.openai.api.getBaseURL = mockGetBaseURL; + + const context = new ProviderContext('openai'); + const result = await context.getBaseURL(mockRequestContext); + + expect(result).toBe(customURL); + }); + }); + + describe('getEndpointPath', () => { + it('should call provider getEndpoint function with correct parameters', () => { + const mockEndpoint = '/v1/chat/completions'; + const mockGetEndpoint = jest.fn().mockReturnValue(mockEndpoint); + Providers.openai.api.getEndpoint = mockGetEndpoint; + + const context = new ProviderContext('openai'); + const result = context.getEndpointPath(mockRequestContext); + + expect(mockGetEndpoint).toHaveBeenCalledWith({ + c: mockRequestContext.honoContext, + providerOptions: mockRequestContext.providerOption, + fn: mockRequestContext.endpoint, + gatewayRequestBodyJSON: mockRequestContext.params, + gatewayRequestBody: {}, + gatewayRequestURL: mockRequestContext.honoContext.req.url, + }); + expect(result).toBe(mockEndpoint); + }); + }); + + describe('getProxyPath', () => { + it('should handle regular proxy path construction', () => { + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { + url: 'https://gateway.example.com/v1/proxy/chat/completions?model=gpt-4', + }, + }, + } as RequestContext; + + const context = new ProviderContext('openai'); + const result = context.getProxyPath( + mockContext, + 'https://api.openai.com' + ); + + expect(result).toBe( + 'https://api.openai.com/chat/completions?model=gpt-4' + ); + }); + + it('should handle Azure OpenAI special case', () => { + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { + url: 'https://gateway.example.com/v1/proxy/myresource.openai.azure.com/openai/deployments/gpt-4/chat/completions', + }, + }, + } as RequestContext; + + const context = new ProviderContext(AZURE_OPEN_AI); + const result = context.getProxyPath( + mockContext, + 'https://api.openai.com' + ); + + expect(result).toBe( + 'https://myresource.openai.azure.com/openai/deployments/gpt-4/chat/completions' + ); + }); + + it('should use provider-specific getProxyEndpoint when available', () => { + const mockGetProxyEndpoint = jest + .fn() + .mockReturnValue('/custom/endpoint'); + Providers.openai.api.getProxyEndpoint = mockGetProxyEndpoint; + + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { url: 'https://gateway.example.com/v1/proxy/chat/completions' }, + }, + } as RequestContext; + + const context = new ProviderContext('openai'); + const result = context.getProxyPath( + mockContext, + 'https://api.openai.com' + ); + + expect(mockGetProxyEndpoint).toHaveBeenCalledWith({ + reqPath: '/chat/completions', + reqQuery: '', + providerOptions: mockRequestContext.providerOption, + }); + expect(result).toBe('https://api.openai.com/custom/endpoint'); + + // Clean up the mock + delete Providers.openai.api.getProxyEndpoint; + }); + + it('should handle Anthropic double v1 path fix', () => { + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { url: 'https://gateway.example.com/v1/v1/messages' }, + }, + } as RequestContext; + + const context = new ProviderContext(ANTHROPIC); + const result = context.getProxyPath( + mockContext, + 'https://api.anthropic.com' + ); + + expect(result).toBe('https://api.anthropic.com/v1/messages'); + }); + + it('should handle /v1 proxy endpoint path', () => { + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { url: 'https://gateway.example.com/v1/chat/completions' }, + }, + } as RequestContext; + + const context = new ProviderContext('openai'); + const result = context.getProxyPath( + mockContext, + 'https://api.openai.com' + ); + + expect(result).toBe('https://api.openai.com/chat/completions'); + }); + + it('should handle query parameters', () => { + const mockContext = { + ...mockRequestContext, + honoContext: { + req: { + url: 'https://gateway.example.com/v1/proxy/files?purpose=fine-tune&limit=10', + }, + }, + } as RequestContext; + + const context = new ProviderContext('openai'); + const result = context.getProxyPath( + mockContext, + 'https://api.openai.com' + ); + + expect(result).toBe( + 'https://api.openai.com/files?purpose=fine-tune&limit=10' + ); + }); + }); + + describe('getFullURL', () => { + it('should return proxy path for proxy endpoint', async () => { + const mockContext = { + ...mockRequestContext, + endpoint: 'proxy', + customHost: '', + honoContext: { + req: { url: 'https://gateway.example.com/v1/proxy/chat/completions' }, + }, + } as RequestContext; + + const mockGetBaseURL = jest + .fn() + .mockResolvedValue('https://api.openai.com'); + Providers.openai.api.getBaseURL = mockGetBaseURL; + + const context = new ProviderContext('openai'); + const result = await context.getFullURL(mockContext); + + expect(result).toBe('https://api.openai.com/chat/completions'); + }); + + it('should return standard endpoint URL for non-proxy endpoints', async () => { + const mockGetBaseURL = jest + .fn() + .mockResolvedValue('https://api.openai.com'); + const mockGetEndpoint = jest.fn().mockReturnValue('/v1/chat/completions'); + Providers.openai.api.getBaseURL = mockGetBaseURL; + Providers.openai.api.getEndpoint = mockGetEndpoint; + + const context = new ProviderContext('openai'); + const result = await context.getFullURL(mockRequestContext); + + expect(result).toBe('https://api.openai.com/v1/chat/completions'); + }); + + it('should use custom host when provided', async () => { + const mockContext = { + ...mockRequestContext, + customHost: 'https://custom.openai.com', + } as RequestContext; + + const mockGetEndpoint = jest.fn().mockReturnValue('/v1/chat/completions'); + Providers.openai.api.getEndpoint = mockGetEndpoint; + + const context = new ProviderContext('openai'); + const result = await context.getFullURL(mockContext); + + expect(result).toBe('https://custom.openai.com/v1/chat/completions'); + }); + }); + + describe('requestHandlers getter', () => { + it('should return request handlers from provider config', () => { + const context = new ProviderContext('openai'); + expect(context.requestHandlers).toBe(Providers.openai.requestHandlers); + }); + + it('should return empty object when no request handlers', () => { + const context = new ProviderContext('anthropic'); + expect(context.requestHandlers).toEqual({}); + }); + }); + + describe('hasRequestHandler', () => { + it('should return true when handler exists', () => { + const mockContext = { + ...mockRequestContext, + endpoint: 'uploadFile', + } as RequestContext; + + const context = new ProviderContext('openai'); + expect(context.hasRequestHandler(mockContext)).toBe(true); + }); + + it('should return false when handler does not exist', () => { + const mockContext = { + ...mockRequestContext, + endpoint: 'chatComplete', + } as RequestContext; + + const context = new ProviderContext('openai'); + expect(context.hasRequestHandler(mockContext)).toBe(false); + }); + + it('should return false when no request handlers', () => { + const context = new ProviderContext('anthropic'); + expect(context.hasRequestHandler(mockRequestContext)).toBe(false); + }); + }); + + describe('getRequestHandler', () => { + it('should return wrapped handler function when handler exists', () => { + const mockHandler = jest.fn().mockResolvedValue(new Response('success')); + Providers.openai.requestHandlers!.uploadFile = mockHandler; + + const mockContext = { + ...mockRequestContext, + endpoint: 'uploadFile', + honoContext: { req: { url: 'https://gateway.com/v1/files' } }, + requestHeaders: { authorization: 'Bearer sk-test' }, + requestBody: new FormData(), + } as unknown as RequestContext; + + const context = new ProviderContext('openai'); + const handlerWrapper = context.getRequestHandler(mockContext); + + expect(handlerWrapper).toBeInstanceOf(Function); + }); + + it('should return undefined when handler does not exist', () => { + const mockContext = { + ...mockRequestContext, + endpoint: 'chatComplete', + } as RequestContext; + + const context = new ProviderContext('openai'); + const result = context.getRequestHandler(mockContext); + + expect(result).toBeUndefined(); + }); + + it('should call handler with correct parameters when executed', async () => { + const mockHandler = jest.fn().mockResolvedValue(new Response('success')); + Providers.openai.requestHandlers!.uploadFile = mockHandler; + + const mockContext = { + ...mockRequestContext, + endpoint: 'uploadFile', + honoContext: { req: { url: 'https://gateway.com/v1/files' } }, + requestHeaders: { authorization: 'Bearer sk-test' }, + requestBody: new FormData(), + } as unknown as RequestContext; + + const context = new ProviderContext('openai'); + const handlerWrapper = context.getRequestHandler(mockContext); + + if (handlerWrapper) { + await handlerWrapper(); + + expect(mockHandler).toHaveBeenCalledWith({ + c: mockContext.honoContext, + providerOptions: mockContext.providerOption, + requestURL: mockContext.honoContext.req.url, + requestHeaders: mockContext.requestHeaders, + requestBody: mockContext.requestBody, + }); + } + }); + + it('should return undefined when requestHandlers is undefined', () => { + // Create a provider without requestHandlers + Providers['test-provider'] = { + api: { + headers: jest.fn(), + getBaseURL: jest.fn(), + getEndpoint: jest.fn(), + }, + }; + + const context = new ProviderContext('test-provider'); + const result = context.getRequestHandler(mockRequestContext); + + expect(result).toBeUndefined(); + + // Clean up + delete Providers['test-provider']; + }); + }); +}); diff --git a/tests/unit/src/handlers/services/requestContext.test.ts b/tests/unit/src/handlers/services/requestContext.test.ts new file mode 100644 index 0000000..a5bfd7b --- /dev/null +++ b/tests/unit/src/handlers/services/requestContext.test.ts @@ -0,0 +1,805 @@ +import { Context } from 'hono'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import { Options, Params } from '../../../../../src/types/requestBody'; +import { endpointStrings } from '../../../../../src/providers/types'; +import { HEADER_KEYS } from '../../../../../src/globals'; +import { HooksManager } from '../../../../../src/middlewares/hooks'; +import { HookType } from '../../../../../src/middlewares/hooks/types'; + +// Mock the transformToProviderRequest function +jest.mock('../../../services/transformToProviderRequest', () => ({ + transformToProviderRequest: jest.fn().mockReturnValue({ transformed: true }), +})); + +describe('RequestContext', () => { + let mockContext: Context; + let mockProviderOption: Options; + let mockRequestHeaders: Record; + let mockRequestBody: Params; + let requestContext: RequestContext; + + beforeEach(() => { + mockContext = { + get: jest.fn(), + set: jest.fn(), + } as unknown as Context; + + mockProviderOption = { + provider: 'openai', + apiKey: 'sk-test123', + retry: { attempts: 3, onStatusCodes: [500, 502] }, + cache: { mode: 'simple', maxAge: 3600 }, + overrideParams: { temperature: 0.7 }, + forwardHeaders: ['x-custom-header'], + customHost: 'https://custom.openai.com', + requestTimeout: 30000, + strictOpenAiCompliance: true, + beforeRequestHooks: [], + afterRequestHooks: [], + defaultInputGuardrails: [], + defaultOutputGuardrails: [], + }; + + mockRequestHeaders = { + [HEADER_KEYS.CONTENT_TYPE]: 'application/json', + [HEADER_KEYS.TRACE_ID]: 'trace-123', + [HEADER_KEYS.METADATA]: '{"userId": "user123"}', + [HEADER_KEYS.FORWARD_HEADERS]: 'x-custom-header,x-another-header', + [HEADER_KEYS.CUSTOM_HOST]: 'https://custom.api.com', + [HEADER_KEYS.REQUEST_TIMEOUT]: '45000', + [HEADER_KEYS.STRICT_OPEN_AI_COMPLIANCE]: 'true', + authorization: 'Bearer sk-test123', + 'x-custom-header': 'custom-value', + }; + + mockRequestBody = { + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello' }], + stream: false, + }; + + requestContext = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + mockRequestHeaders, + mockRequestBody, + 'POST', + 0 + ); + }); + + describe('constructor', () => { + it('should initialize with provided values', () => { + expect(requestContext.honoContext).toBe(mockContext); + expect(requestContext.providerOption).toBe(mockProviderOption); + expect(requestContext.endpoint).toBe('chatComplete'); + expect(requestContext.requestHeaders).toBe(mockRequestHeaders); + expect(requestContext.requestBody).toBe(mockRequestBody); + expect(requestContext.method).toBe('POST'); + expect(requestContext.index).toBe(0); + }); + + it('should normalize retry config on initialization', () => { + expect(requestContext.providerOption.retry).toEqual({ + attempts: 3, + onStatusCodes: [500, 502], + useRetryAfterHeader: undefined, + }); + }); + + it('should set default retry config when not provided', () => { + const contextWithoutRetry = new RequestContext( + mockContext, + { provider: 'openai' }, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(contextWithoutRetry.providerOption.retry).toEqual({ + attempts: 0, + onStatusCodes: [], + useRetryAfterHeader: undefined, + }); + }); + }); + + describe('requestURL getter/setter', () => { + it('should get and set request URL', () => { + expect(requestContext.requestURL).toBe(''); + + requestContext.requestURL = 'https://api.openai.com/v1/chat/completions'; + expect(requestContext.requestURL).toBe( + 'https://api.openai.com/v1/chat/completions' + ); + }); + }); + + describe('overrideParams getter', () => { + it('should return override params from provider option', () => { + expect(requestContext.overrideParams).toEqual({ temperature: 0.7 }); + }); + + it('should return empty object when no override params', () => { + const context = new RequestContext( + mockContext, + { provider: 'openai' }, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.overrideParams).toEqual({}); + }); + }); + + describe('params getter/setter', () => { + it('should return merged request body and override params', () => { + expect(requestContext.params).toEqual({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello' }], + stream: false, + temperature: 0.7, + }); + }); + + it('should override request body params with override params', () => { + const bodyWithTemperature = { + model: 'gpt-4', + temperature: 0.5, + messages: [], + }; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + bodyWithTemperature, + 'POST', + 0 + ); + + expect(context.params.temperature).toBe(0.7); // Override wins + }); + + it('should return empty object for non-JSON request bodies', () => { + const formData = new FormData(); + const context = new RequestContext( + mockContext, + mockProviderOption, + 'uploadFile' as endpointStrings, + {}, + formData, + 'POST', + 0 + ); + + expect(context.params).toEqual({}); + }); + + it('should allow setting params directly', () => { + requestContext.params = { model: 'gpt-3.5-turbo', messages: [] }; + expect(requestContext.params).toEqual({ + model: 'gpt-3.5-turbo', + messages: [], + }); + }); + + it('should handle ReadableStream request body', () => { + const stream = new ReadableStream(); + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + stream, + 'POST', + 0 + ); + + expect(context.params).toEqual({}); + }); + + it('should handle null request body', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + null as any, + 'POST', + 0 + ); + + expect(context.params).toEqual({}); + }); + }); + + describe('transformedRequestBody getter/setter', () => { + it('should get and set transformed request body', () => { + expect(requestContext.transformedRequestBody).toBeUndefined(); + + const transformed = { model: 'claude-3', messages: [] }; + requestContext.transformedRequestBody = transformed; + expect(requestContext.transformedRequestBody).toBe(transformed); + }); + }); + + describe('getHeader', () => { + it('should return content type without parameters', () => { + const headers = { + [HEADER_KEYS.CONTENT_TYPE.toLowerCase()]: + 'application/json; charset=utf-8', + }; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + headers, + {}, + 'POST', + 0 + ); + + expect(context.getHeader(HEADER_KEYS.CONTENT_TYPE)).toBe( + 'application/json' + ); + }); + + it('should return header value for non-content-type headers', () => { + expect(requestContext.getHeader('authorization')).toBe( + 'Bearer sk-test123' + ); + }); + + it('should return empty string for missing headers', () => { + expect(requestContext.getHeader('non-existent-header')).toBe(''); + }); + }); + + describe('traceId getter', () => { + it('should return trace ID from headers', () => { + expect(requestContext.traceId).toBe('trace-123'); + }); + + it('should return empty string when no trace ID', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.traceId).toBe(''); + }); + }); + + describe('isStreaming getter', () => { + it('should return true when stream is true', () => { + const streamingBody = { ...mockRequestBody, stream: true }; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + streamingBody, + 'POST', + 0 + ); + + expect(context.isStreaming).toBe(true); + }); + + it('should return false when stream is false', () => { + expect(requestContext.isStreaming).toBe(false); + }); + + it('should return false when stream is not set', () => { + const { stream, ...bodyWithoutStream } = mockRequestBody; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + bodyWithoutStream, + 'POST', + 0 + ); + + expect(context.isStreaming).toBe(false); + }); + }); + + describe('strictOpenAiCompliance getter', () => { + it('should return false when header is "false"', () => { + const headers = { + [HEADER_KEYS.STRICT_OPEN_AI_COMPLIANCE]: 'false', + }; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + headers, + {}, + 'POST', + 0 + ); + + expect(context.strictOpenAiCompliance).toBe(false); + }); + + it('should return false when provider option is false', () => { + const option = { ...mockProviderOption, strictOpenAiCompliance: false }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.strictOpenAiCompliance).toBe(false); + }); + + it('should return true by default', () => { + const context = new RequestContext( + mockContext, + { provider: 'openai' }, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.strictOpenAiCompliance).toBe(true); + }); + }); + + describe('metadata getter', () => { + it('should parse JSON metadata from headers', () => { + expect(requestContext.metadata).toEqual({ userId: 'user123' }); + }); + + it('should return empty object for invalid JSON', () => { + const headers = { + [HEADER_KEYS.METADATA]: '{invalid json}', + }; + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + headers, + {}, + 'POST', + 0 + ); + + expect(context.metadata).toEqual({}); + }); + + it('should return empty object when no metadata header', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.metadata).toEqual({}); + }); + }); + + describe('forwardHeaders getter', () => { + it('should parse forward headers from header', () => { + expect(requestContext.forwardHeaders).toEqual([ + 'x-custom-header', + 'x-another-header', + ]); + }); + + it('should return forward headers from provider option when header not present', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.forwardHeaders).toEqual(['x-custom-header']); + }); + + it('should return empty array when neither header nor option present', () => { + const option = { ...mockProviderOption }; + delete option.forwardHeaders; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.forwardHeaders).toEqual([]); + }); + }); + + describe('customHost getter', () => { + it('should return custom host from header', () => { + expect(requestContext.customHost).toBe('https://custom.api.com'); + }); + + it('should return custom host from provider option when header not present', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.customHost).toBe('https://custom.openai.com'); + }); + + it('should return empty string when neither present', () => { + const option = { ...mockProviderOption }; + delete option.customHost; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.customHost).toBe(''); + }); + }); + + describe('requestTimeout getter', () => { + it('should return timeout from header as number', () => { + expect(requestContext.requestTimeout).toBe(45000); + }); + + it('should return timeout from provider option when header not present', () => { + const context = new RequestContext( + mockContext, + mockProviderOption, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.requestTimeout).toBe(30000); + }); + + it('should return null when neither present', () => { + const option = { ...mockProviderOption }; + delete option.requestTimeout; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.requestTimeout).toBeNull(); + }); + }); + + describe('provider getter', () => { + it('should return provider from provider option', () => { + expect(requestContext.provider).toBe('openai'); + }); + + it('should return empty string when no provider', () => { + const context = new RequestContext( + mockContext, + { provider: 'openai' }, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.provider).toBe(''); + }); + }); + + describe('retryConfig getter', () => { + it('should return normalized retry config', () => { + expect(requestContext.retryConfig).toEqual({ + attempts: 3, + onStatusCodes: [500, 502], + useRetryAfterHeader: undefined, + }); + }); + }); + + describe('cacheConfig getter', () => { + it('should return cache config from object', () => { + expect(requestContext.cacheConfig).toEqual({ + mode: 'simple', + maxAge: 3600, + cacheStatus: 'MISS', + }); + }); + + it('should return cache config from string', () => { + const option = { ...mockProviderOption, cache: 'semantic' }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.cacheConfig).toEqual({ + mode: 'semantic', + maxAge: undefined, + cacheStatus: 'MISS', + }); + }); + + it('should return disabled cache when no config', () => { + const option = { ...mockProviderOption }; + delete option.cache; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.cacheConfig).toEqual({ + mode: 'DISABLED', + maxAge: undefined, + cacheStatus: 'DISABLED', + }); + }); + + it('should parse string maxAge to number', () => { + const option = { + ...mockProviderOption, + cache: { mode: 'simple', maxAge: 7200 }, + }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.cacheConfig.maxAge).toBe(7200); + }); + }); + + describe('hasRetries', () => { + it('should return true when retry attempts > 0', () => { + expect(requestContext.hasRetries()).toBe(true); + }); + + it('should return false when retry attempts = 0', () => { + const option = { + ...mockProviderOption, + retry: { attempts: 0, onStatusCodes: [] }, + }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.hasRetries()).toBe(false); + }); + }); + + describe('hooks getters', () => { + it('should return combined before request hooks', () => { + const beforeHooks = [ + { + id: 'hook1', + type: HookType.GUARDRAIL, + eventType: 'beforeRequestHook' as const, + }, + ]; + const defaultInputGuardrails = [ + { + id: 'guard1', + type: HookType.GUARDRAIL, + eventType: 'beforeRequestHook' as const, + }, + ]; + const option = { + ...mockProviderOption, + beforeRequestHooks: beforeHooks, + defaultInputGuardrails: defaultInputGuardrails, + }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.beforeRequestHooks).toEqual([ + ...beforeHooks, + ...defaultInputGuardrails, + ]); + }); + + it('should return combined after request hooks', () => { + const afterHooks = [ + { + id: 'hook2', + type: HookType.GUARDRAIL, + eventType: 'afterRequestHook' as const, + }, + ]; + const defaultOutputGuardrails = [ + { + id: 'guard2', + type: HookType.GUARDRAIL, + eventType: 'afterRequestHook' as const, + }, + ]; + const option = { + ...mockProviderOption, + afterRequestHooks: afterHooks, + defaultOutputGuardrails: defaultOutputGuardrails, + }; + const context = new RequestContext( + mockContext, + option, + 'chatComplete' as endpointStrings, + {}, + {}, + 'POST', + 0 + ); + + expect(context.afterRequestHooks).toEqual([ + ...afterHooks, + ...defaultOutputGuardrails, + ]); + }); + }); + + describe('hooksManager getter', () => { + it('should return hooks manager from context', () => { + const mockHooksManager = {} as HooksManager; + (mockContext.get as jest.Mock).mockReturnValue(mockHooksManager); + + expect(requestContext.hooksManager).toBe(mockHooksManager); + expect(mockContext.get).toHaveBeenCalledWith('hooksManager'); + }); + }); + + describe('transformToProviderRequestAndSave', () => { + it('should transform request body for POST method', () => { + const { + transformToProviderRequest, + } = require('../../../services/transformToProviderRequest'); + + requestContext.transformToProviderRequestAndSave(); + + expect(transformToProviderRequest).toHaveBeenCalledWith( + 'openai', + requestContext.params, + requestContext.requestBody, + 'chatComplete', + requestContext.requestHeaders, + requestContext.providerOption + ); + expect(requestContext.transformedRequestBody).toEqual({ + transformed: true, + }); + }); + + it('should not transform for non-POST methods', () => { + const { + transformToProviderRequest, + } = require('../../../services/transformToProviderRequest'); + const context = new RequestContext( + mockContext, + mockProviderOption, + 'listFiles' as endpointStrings, + {}, + mockRequestBody, + 'GET', + 0 + ); + + context.transformToProviderRequestAndSave(); + + expect(transformToProviderRequest).not.toHaveBeenCalled(); + expect(context.transformedRequestBody).toBe(mockRequestBody); + }); + }); + + describe('requestOptions getter/setter', () => { + it('should get request options from context', () => { + const mockOptions = [{ option1: 'value1' }]; + (mockContext.get as jest.Mock).mockReturnValue(mockOptions); + + expect(requestContext.requestOptions).toBe(mockOptions); + expect(mockContext.get).toHaveBeenCalledWith('requestOptions'); + }); + + it('should return empty array when no options', () => { + (mockContext.get as jest.Mock).mockReturnValue(undefined); + + expect(requestContext.requestOptions).toEqual([]); + }); + }); + + describe('appendRequestOptions', () => { + it('should append request options to existing options', () => { + const existingOptions = [{ option1: 'value1' }]; + const newOption = { option2: 'value2' }; + (mockContext.get as jest.Mock).mockReturnValue(existingOptions); + + requestContext.appendRequestOptions(newOption); + + expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ + { option1: 'value1' }, + { option2: 'value2' }, + ]); + }); + + it('should append to empty options array', () => { + const newOption = { option1: 'value1' }; + (mockContext.get as jest.Mock).mockReturnValue([]); + + requestContext.appendRequestOptions(newOption); + + expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ + { option1: 'value1' }, + ]); + }); + }); +}); diff --git a/tests/unit/src/handlers/services/responseService.test.ts b/tests/unit/src/handlers/services/responseService.test.ts new file mode 100644 index 0000000..597b480 --- /dev/null +++ b/tests/unit/src/handlers/services/responseService.test.ts @@ -0,0 +1,498 @@ +import { ResponseService } from '../../../../../src/handlers/services/responseService'; +import { RequestContext } from '../../../../../src/handlers/services/requestContext'; +import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; +import { HooksService } from '../../../../../src/handlers/services/hooksService'; +import { LogsService } from '../../../../../src/handlers/services/logsService'; +import { responseHandler } from '../../../../../src/handlers/responseHandlers'; +import { getRuntimeKey } from 'hono/adapter'; +import { + RESPONSE_HEADER_KEYS, + HEADER_KEYS, + POWERED_BY, +} from '../../../../../src/globals'; + +// Mock dependencies +jest.mock('../../responseHandlers'); +jest.mock('hono/adapter'); + +describe('ResponseService', () => { + let mockRequestContext: RequestContext; + let mockProviderContext: ProviderContext; + let mockHooksService: HooksService; + let mockLogsService: LogsService; + let responseService: ResponseService; + + beforeEach(() => { + mockRequestContext = { + index: 0, + traceId: 'trace-123', + provider: 'openai', + isStreaming: false, + params: { model: 'gpt-4', messages: [] }, + strictOpenAiCompliance: true, + requestURL: 'https://api.openai.com/v1/chat/completions', + honoContext: { + req: { url: 'https://gateway.com/v1/chat/completions' }, + }, + } as unknown as RequestContext; + + mockProviderContext = {} as ProviderContext; + + mockHooksService = { + areSyncHooksAvailable: false, + } as unknown as HooksService; + + mockLogsService = {} as LogsService; + + responseService = new ResponseService( + mockRequestContext, + mockProviderContext, + mockHooksService, + mockLogsService + ); + + // Reset mocks + jest.clearAllMocks(); + (getRuntimeKey as jest.Mock).mockReturnValue('node'); + }); + + describe('create', () => { + let mockResponse: Response; + + beforeEach(() => { + mockResponse = new Response( + JSON.stringify({ choices: [{ message: { content: 'Hello' } }] }), + { + status: 200, + headers: { + 'content-type': 'application/json', + 'content-encoding': 'gzip', + 'content-length': '100', + 'transfer-encoding': 'chunked', + }, + } + ); + }); + + it('should create response for already mapped response', async () => { + const options = { + response: mockResponse, + responseTransformer: undefined, + isResponseAlreadyMapped: true, + cache: { + isCacheHit: false, + cacheStatus: 'MISS', + cacheKey: 'cache-key-123', + }, + retryAttempt: 0, + originalResponseJson: { choices: [{ message: { content: 'Hello' } }] }, + }; + + const result = await responseService.create(options); + + expect(result.response).toBe(mockResponse); + expect(result.originalResponseJson).toEqual({ + choices: [{ message: { content: 'Hello' } }], + }); + + // Check headers were updated + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.LAST_USED_OPTION_INDEX) + ).toBe('0'); + expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.TRACE_ID)).toBe( + 'trace-123' + ); + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.RETRY_ATTEMPT_COUNT) + ).toBe('0'); + expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBe('openai'); + }); + + it('should create response for non-mapped response', async () => { + const mappedResponse = new Response('{"mapped": true}', { status: 200 }); + const originalJson = { original: true }; + const responseJson = { response: true }; + + (responseHandler as jest.Mock).mockResolvedValue({ + response: mappedResponse, + originalResponseJson: originalJson, + responseJson: responseJson, + }); + + const options = { + response: mockResponse, + responseTransformer: 'chatComplete', + isResponseAlreadyMapped: false, + cache: { + isCacheHit: false, + cacheStatus: 'MISS', + cacheKey: undefined, + }, + retryAttempt: 1, + }; + + const result = await responseService.create(options); + + expect(responseHandler).toHaveBeenCalledWith( + mockResponse, + mockRequestContext.isStreaming, + mockRequestContext.provider, + 'chatComplete', + mockRequestContext.requestURL, + false, + mockRequestContext.params, + mockRequestContext.strictOpenAiCompliance, + mockRequestContext.honoContext.req.url, + mockHooksService.areSyncHooksAvailable + ); + + expect(result.response).toEqual(mockResponse); + expect(result.responseJson).toBe(responseJson); + expect(result.originalResponseJson).toBe(originalJson); + }); + + it('should handle cache hit scenario', async () => { + const options = { + response: mockResponse, + responseTransformer: 'chatComplete', + isResponseAlreadyMapped: false, + cache: { + isCacheHit: true, + cacheStatus: 'HIT', + cacheKey: 'cache-key-456', + }, + retryAttempt: 0, + }; + + (responseHandler as jest.Mock).mockResolvedValue({ + response: mockResponse, + originalResponseJson: null, + responseJson: null, + }); + + const result = await responseService.create(options); + + expect(responseHandler).toHaveBeenCalledWith( + mockResponse, + mockRequestContext.isStreaming, + mockRequestContext.provider, + 'chatComplete', + mockRequestContext.requestURL, + true, // isCacheHit should be true + mockRequestContext.params, + mockRequestContext.strictOpenAiCompliance, + mockRequestContext.honoContext.req.url, + mockHooksService.areSyncHooksAvailable + ); + + expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS)).toBe( + 'HIT' + ); + }); + + it('should throw error for non-ok response', async () => { + const errorResponse = new Response('{"error": "Bad Request"}', { + status: 400, + }); + const options = { + response: errorResponse, + responseTransformer: undefined, + isResponseAlreadyMapped: true, + cache: { + isCacheHit: false, + cacheStatus: 'MISS', + cacheKey: undefined, + }, + retryAttempt: 0, + }; + + await expect(responseService.create(options)).rejects.toThrow(); + }); + + it('should handle error response correctly', async () => { + const errorResponse = new Response('{"error": "Internal Server Error"}', { + status: 500, + }); + const options = { + response: errorResponse, + responseTransformer: undefined, + isResponseAlreadyMapped: true, + cache: { + isCacheHit: false, + cacheStatus: 'MISS', + cacheKey: undefined, + }, + retryAttempt: 0, + }; + + try { + await responseService.create(options); + } catch (error: any) { + expect(error.status).toBe(500); + expect(error.response).toBe(errorResponse); + expect(error.message).toBe('{"error": "Internal Server Error"}'); + } + }); + + it('should not add cache status header when not provided', async () => { + const options = { + response: mockResponse, + responseTransformer: undefined, + isResponseAlreadyMapped: true, + cache: { + isCacheHit: false, + cacheStatus: undefined, + cacheKey: undefined, + }, + retryAttempt: 0, + }; + + await responseService.create(options); + + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS) + ).toBeNull(); + }); + + it('should not add provider header when provider is POWERED_BY', async () => { + const contextWithPortkey = { + ...mockRequestContext, + provider: POWERED_BY, + } as RequestContext; + + const serviceWithPortkey = new ResponseService( + contextWithPortkey, + mockProviderContext, + mockHooksService, + mockLogsService + ); + + const options = { + response: mockResponse, + responseTransformer: undefined, + isResponseAlreadyMapped: true, + cache: { + isCacheHit: false, + cacheStatus: 'MISS', + cacheKey: undefined, + }, + retryAttempt: 0, + }; + + await serviceWithPortkey.create(options); + + expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); + }); + }); + + describe('getResponse', () => { + it('should call responseHandler with correct parameters', async () => { + const mockResponse = new Response('{}'); + const expectedResult = { + response: mockResponse, + originalResponseJson: { test: true }, + responseJson: { response: true }, + }; + + (responseHandler as jest.Mock).mockResolvedValue(expectedResult); + + const result = await responseService.getResponse( + mockResponse, + 'chatComplete', + false + ); + + expect(responseHandler).toHaveBeenCalledWith( + mockResponse, + mockRequestContext.isStreaming, + mockRequestContext.provider, + 'chatComplete', + mockRequestContext.requestURL, + false, + mockRequestContext.params, + mockRequestContext.strictOpenAiCompliance, + mockRequestContext.honoContext.req.url, + mockHooksService.areSyncHooksAvailable + ); + + expect(result).toBe(expectedResult); + }); + + it('should handle streaming responses', async () => { + const streamingContext = { + ...mockRequestContext, + isStreaming: true, + } as RequestContext; + + const streamingService = new ResponseService( + streamingContext, + mockProviderContext, + mockHooksService, + mockLogsService + ); + + const mockResponse = new Response('{}'); + (responseHandler as jest.Mock).mockResolvedValue({ + response: mockResponse, + originalResponseJson: null, + responseJson: null, + }); + + await streamingService.getResponse(mockResponse, 'chatComplete', false); + + expect(responseHandler).toHaveBeenCalledWith( + mockResponse, + true, // isStreaming should be true + streamingContext.provider, + 'chatComplete', + streamingContext.requestURL, + false, + streamingContext.params, + streamingContext.strictOpenAiCompliance, + streamingContext.honoContext.req.url, + mockHooksService.areSyncHooksAvailable + ); + }); + + it('should handle cache hit scenario', async () => { + const mockResponse = new Response('{}'); + (responseHandler as jest.Mock).mockResolvedValue({ + response: mockResponse, + originalResponseJson: null, + responseJson: null, + }); + + await responseService.getResponse(mockResponse, 'chatComplete', true); + + expect(responseHandler).toHaveBeenCalledWith( + mockResponse, + mockRequestContext.isStreaming, + mockRequestContext.provider, + 'chatComplete', + mockRequestContext.requestURL, + true, // isCacheHit should be true + mockRequestContext.params, + mockRequestContext.strictOpenAiCompliance, + mockRequestContext.honoContext.req.url, + mockHooksService.areSyncHooksAvailable + ); + }); + }); + + describe('updateHeaders', () => { + let mockResponse: Response; + + beforeEach(() => { + mockResponse = new Response('{}', { + headers: { + 'content-encoding': 'br, gzip', + 'content-length': '100', + 'transfer-encoding': 'chunked', + }, + }); + }); + + it('should add required headers', () => { + responseService.updateHeaders(mockResponse, 'HIT', 2); + + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.LAST_USED_OPTION_INDEX) + ).toBe('0'); + expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.TRACE_ID)).toBe( + 'trace-123' + ); + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.RETRY_ATTEMPT_COUNT) + ).toBe('2'); + expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS)).toBe( + 'HIT' + ); + expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBe('openai'); + }); + + it('should remove problematic headers', () => { + responseService.updateHeaders(mockResponse, undefined, 0); + + expect(mockResponse.headers.get('content-length')).toBeNull(); + expect(mockResponse.headers.get('transfer-encoding')).toBeNull(); + }); + + it('should remove brotli encoding', () => { + responseService.updateHeaders(mockResponse, undefined, 0); + + expect(mockResponse.headers.get('content-encoding')).toBeNull(); + }); + + it('should remove content-encoding for node runtime', () => { + (getRuntimeKey as jest.Mock).mockReturnValue('node'); + const response = new Response('{}', { + headers: { 'content-encoding': 'gzip' }, + }); + + responseService.updateHeaders(response, undefined, 0); + + expect(response.headers.get('content-encoding')).toBeNull(); + }); + + it('should keep content-encoding for non-brotli, non-node', () => { + (getRuntimeKey as jest.Mock).mockReturnValue('workerd'); + const response = new Response('{}', { + headers: { 'content-encoding': 'gzip' }, + }); + + responseService.updateHeaders(response, undefined, 0); + + expect(response.headers.get('content-encoding')).toBe('gzip'); + }); + + it('should not add cache status header when undefined', () => { + responseService.updateHeaders(mockResponse, undefined, 0); + + expect( + mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS) + ).toBeNull(); + }); + + it('should not add provider header when provider is POWERED_BY', () => { + const contextWithPortkey = { + ...mockRequestContext, + provider: POWERED_BY, + } as RequestContext; + + const serviceWithPortkey = new ResponseService( + contextWithPortkey, + mockProviderContext, + mockHooksService, + mockLogsService + ); + + serviceWithPortkey.updateHeaders(mockResponse, 'MISS', 0); + + expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); + }); + + it('should not add provider header when provider is empty', () => { + const contextWithEmptyProvider = { + ...mockRequestContext, + provider: '', + } as RequestContext; + + const serviceWithEmptyProvider = new ResponseService( + contextWithEmptyProvider, + mockProviderContext, + mockHooksService, + mockLogsService + ); + + serviceWithEmptyProvider.updateHeaders(mockResponse, 'MISS', 0); + + expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); + }); + + it('should return the response object', () => { + const result = responseService.updateHeaders(mockResponse, 'MISS', 0); + + expect(result).toBe(mockResponse); + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..880fc95 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true, + "skipLibCheck": true, + "lib": ["esnext"], + "types": ["@cloudflare/workers-types", "node", "jest"], + "jsx": "react-jsx", + "jsxImportSource": "hono/jsx", + "resolveJsonModule": true + }, + "exclude": ["**/*.test.ts", "cookbook"] +} diff --git a/wrangler.toml b/wrangler.toml new file mode 100644 index 0000000..3784967 --- /dev/null +++ b/wrangler.toml @@ -0,0 +1,30 @@ +name = "rubeus" +compatibility_date = "2024-12-05" +main = "src/index.ts" +compatibility_flags = [ "nodejs_compat" ] + +[vars] +ENVIRONMENT = 'dev' +CUSTOM_HEADERS_TO_IGNORE = [] + +# +#Configuration for DEVELOPMENT environment +# +[env.staging] +name = "rubeus-dev" + +[env.staging.vars] +ENVIRONMENT = 'staging' +CUSTOM_HEADERS_TO_IGNORE = [] + + +# +#Configuration for PRODUCTION environment +# +[env.production] +name = "rubeus" +logpush=true + +[env.production.vars] +ENVIRONMENT = 'production' +CUSTOM_HEADERS_TO_IGNORE = []