chore: import upstream snapshot with attribution

apps/sim/providers/vllm/index.test.ts

@@ -0,0 +1,398 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockCreate,
+  openAIArgs,
+  mockOpenAI,
+  mockExecuteTool,
+  mockPrepareTools,
+  mockCheckForced,
+  mockCreateStream,
+  mockValidateUrlWithDNS,
+  mockCreatePinnedFetch,
+  pinnedFetchFn,
+  envState,
+} = vi.hoisted(() => {
+  const openAIArgs: Array<Record<string, unknown>> = []
+  const mockCreate = vi.fn()
+  const pinnedFetchFn = vi.fn()
+  class MockOpenAI {
+    chat = { completions: { create: mockCreate } }
+    constructor(opts: Record<string, unknown>) {
+      openAIArgs.push(opts)
+    }
+  }
+  return {
+    mockCreate,
+    openAIArgs,
+    mockOpenAI: MockOpenAI,
+    mockExecuteTool: vi.fn(),
+    mockPrepareTools: vi.fn(),
+    mockCheckForced: vi.fn(),
+    mockCreateStream: vi.fn(),
+    mockValidateUrlWithDNS: vi.fn(),
+    mockCreatePinnedFetch: vi.fn(() => pinnedFetchFn),
+    pinnedFetchFn,
+    envState: {
+      VLLM_BASE_URL: 'http://localhost:8000',
+      VLLM_API_KEY: undefined as string | undefined,
+    },
+  }
+})
+
+vi.mock('openai', () => ({ default: mockOpenAI }))
+vi.mock('@/lib/core/config/env', () => ({ env: envState }))
+vi.mock('@/lib/core/security/input-validation.server', () => ({
+  validateUrlWithDNS: mockValidateUrlWithDNS,
+  createPinnedFetch: mockCreatePinnedFetch,
+}))
+vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 20 }))
+vi.mock('@/providers/models', () => ({
+  getProviderFileAttachment: vi
+    .fn()
+    .mockReturnValue({ maxBytes: 10 * 1024 * 1024, strategy: 'inline' }),
+  INLINE_ATTACHMENT_MAX_BYTES: 10 * 1024 * 1024,
+  getProviderModels: vi.fn(() => []),
+  getProviderDefaultModel: vi.fn(() => 'vllm/generic'),
+}))
+vi.mock('@/providers/attachments', () => ({
+  formatMessagesForProvider: vi.fn((messages) => messages),
+}))
+vi.mock('@/providers/trace-enrichment', () => ({
+  enrichLastModelSegmentFromChatCompletions: vi.fn(),
+}))
+vi.mock('@/providers/utils', () => ({
+  calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })),
+  prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })),
+  prepareToolsWithUsageControl: mockPrepareTools,
+  sumToolCosts: vi.fn(() => 0),
+}))
+vi.mock('@/providers/vllm/utils', () => ({
+  checkForForcedToolUsage: mockCheckForced,
+  createReadableStreamFromVLLMStream: mockCreateStream,
+}))
+vi.mock('@/tools', () => ({ executeTool: mockExecuteTool }))
+vi.mock('@/stores/providers', () => ({
+  useProvidersStore: { getState: () => ({ setProviderModels: vi.fn() }) },
+}))
+
+import { clearProviderClientCacheForTests } from '@/providers/client-cache'
+import type { ProviderToolConfig } from '@/providers/types'
+import { vllmProvider } from '@/providers/vllm/index'
+
+interface ToolCall {
+  id: string
+  type: 'function'
+  function: { name: string; arguments: string }
+}
+
+function chatResponse(content: string | null, toolCalls?: ToolCall[]) {
+  return {
+    choices: [{ message: { content, tool_calls: toolCalls } }],
+    usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+  }
+}
+
+function makeTool(id: string): ProviderToolConfig {
+  return {
+    id,
+    name: id,
+    description: '',
+    params: {},
+    parameters: { type: 'object', properties: {}, required: [] },
+  }
+}
+
+const toolCall = (id: string, name: string, args = '{}'): ToolCall => ({
+  id,
+  type: 'function',
+  function: { name, arguments: args },
+})
+
+/** Payload passed to the Nth `chat.completions.create` call. */
+const createPayload = (callIndex: number) => mockCreate.mock.calls[callIndex][0]
+
+describe('vllmProvider', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    clearProviderClientCacheForTests()
+    openAIArgs.length = 0
+    envState.VLLM_BASE_URL = 'http://localhost:8000'
+    envState.VLLM_API_KEY = undefined
+    mockPrepareTools.mockReturnValue({
+      tools: [{ type: 'function', function: { name: 'myTool' } }],
+      toolChoice: 'auto',
+      forcedTools: [],
+      hasFilteredTools: false,
+    })
+    mockCheckForced.mockReturnValue({ hasUsedForcedTool: false, usedForcedTools: [] })
+    mockCreateStream.mockReturnValue(new ReadableStream({ start: (c) => c.close() }))
+    mockExecuteTool.mockResolvedValue({ success: true, output: { result: 'ok' } })
+    mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' })
+    mockCreatePinnedFetch.mockReturnValue(pinnedFetchFn)
+  })
+
+  describe('endpoint SSRF protection', () => {
+    it('does not validate or pin when no endpoint is supplied (uses env base URL)', async () => {
+      mockCreate.mockResolvedValueOnce(chatResponse('hi'))
+
+      await vllmProvider.executeRequest({
+        model: 'vllm/llama-3',
+        messages: [{ role: 'user', content: 'hi' }],
+      })
+
+      expect(mockValidateUrlWithDNS).not.toHaveBeenCalled()
+      expect(mockCreatePinnedFetch).not.toHaveBeenCalled()
+      expect(openAIArgs[0].baseURL).toBe('http://localhost:8000/v1')
+      expect(openAIArgs[0].fetch).toBeUndefined()
+    })
+
+    it('validates a user-supplied endpoint and pins the connection to the resolved IP', async () => {
+      mockCreate.mockResolvedValueOnce(chatResponse('hi'))
+
+      await vllmProvider.executeRequest({
+        model: 'vllm/llama-3',
+        messages: [{ role: 'user', content: 'hi' }],
+        azureEndpoint: 'https://my-vllm.example.com',
+      })
+
+      expect(mockValidateUrlWithDNS).toHaveBeenCalledWith(
+        'https://my-vllm.example.com',
+        'vLLM endpoint',
+        { allowHttp: true }
+      )
+      expect(mockCreatePinnedFetch).toHaveBeenCalledWith('203.0.113.10')
+      expect(openAIArgs[0].baseURL).toBe('https://my-vllm.example.com/v1')
+      expect(openAIArgs[0].fetch).toBe(pinnedFetchFn)
+    })
+
+    it('rejects a user-supplied endpoint that fails SSRF validation without issuing a request', async () => {
+      mockValidateUrlWithDNS.mockResolvedValueOnce({
+        isValid: false,
+        error: 'vLLM endpoint resolves to a blocked IP address',
+      })
+
+      await expect(
+        vllmProvider.executeRequest({
+          model: 'vllm/llama-3',
+          messages: [{ role: 'user', content: 'hi' }],
+          azureEndpoint: 'http://169.254.169.254',
+        })
+      ).rejects.toThrow('Invalid vLLM endpoint')
+
+      expect(mockCreatePinnedFetch).not.toHaveBeenCalled()
+      expect(openAIArgs).toHaveLength(0)
+      expect(mockCreate).not.toHaveBeenCalled()
+    })
+
+    it('rejects a validated endpoint that did not resolve to a pinnable IP', async () => {
+      mockValidateUrlWithDNS.mockResolvedValueOnce({ isValid: true })
+
+      await expect(
+        vllmProvider.executeRequest({
+          model: 'vllm/llama-3',
+          messages: [{ role: 'user', content: 'hi' }],
+          azureEndpoint: 'https://my-vllm.example.com',
+        })
+      ).rejects.toThrow('could not resolve a pinnable IP address')
+
+      expect(mockCreatePinnedFetch).not.toHaveBeenCalled()
+      expect(openAIArgs).toHaveLength(0)
+      expect(mockCreate).not.toHaveBeenCalled()
+    })
+  })
+
+  it('builds a chat payload with the vllm/ prefix stripped and messages assembled in order', async () => {
+    mockCreate.mockResolvedValueOnce(chatResponse('hello'))
+
+    const result = await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      systemPrompt: 'be helpful',
+      context: 'prior context',
+      messages: [{ role: 'user', content: 'hi' }],
+      temperature: 0.7,
+      maxTokens: 256,
+    })
+
+    const payload = createPayload(0)
+    expect(payload.model).toBe('llama-3')
+    expect(payload.temperature).toBe(0.7)
+    expect(payload.max_completion_tokens).toBe(256)
+    expect(payload.messages.map((m: { role: string }) => m.role)).toEqual([
+      'system',
+      'user',
+      'user',
+    ])
+    expect(result.content).toBe('hello')
+    expect(result.tokens).toEqual({ input: 10, output: 5, total: 15 })
+  })
+
+  it('sends response_format as json_schema with strict when a responseFormat is provided', async () => {
+    mockCreate.mockResolvedValueOnce(chatResponse('{}'))
+
+    await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'hi' }],
+      responseFormat: { name: 'out', schema: { type: 'object' }, strict: true },
+    })
+
+    expect(createPayload(0).response_format).toEqual({
+      type: 'json_schema',
+      json_schema: { name: 'out', schema: { type: 'object' }, strict: true },
+    })
+  })
+
+  it('strips markdown code fences from structured-output content', async () => {
+    mockCreate.mockResolvedValueOnce(chatResponse('```json\n{"a":1}\n```'))
+
+    const result = await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'hi' }],
+      responseFormat: { name: 'out', schema: { type: 'object' }, strict: true },
+    })
+
+    expect(result.content).toBe('{"a":1}')
+  })
+
+  it('runs the tool loop: executes tools, appends assistant + tool messages, returns results', async () => {
+    mockCreate
+      .mockResolvedValueOnce(chatResponse(null, [toolCall('call_1', 'myTool', '{"x":1}')]))
+      .mockResolvedValueOnce(chatResponse('final answer'))
+
+    const result = await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'use a tool' }],
+      tools: [makeTool('myTool')],
+    })
+
+    expect(mockExecuteTool).toHaveBeenCalledWith('myTool', { x: 1 }, expect.anything())
+
+    const [assistantMessage, toolMessage] = createPayload(1).messages.slice(-2)
+    expect(assistantMessage).toMatchObject({
+      role: 'assistant',
+      content: null,
+      tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'myTool' } }],
+    })
+    expect(toolMessage).toMatchObject({ role: 'tool', tool_call_id: 'call_1' })
+    expect(toolMessage).not.toHaveProperty('name')
+
+    expect(result.content).toBe('final answer')
+    expect(result.toolCalls).toHaveLength(1)
+    expect(result.toolCalls?.[0]).toMatchObject({ name: 'myTool', success: true })
+    expect(result.toolResults).toHaveLength(1)
+  })
+
+  it('records a failed tool result without throwing', async () => {
+    mockExecuteTool.mockResolvedValueOnce({ success: false, error: 'tool blew up' })
+    mockCreate
+      .mockResolvedValueOnce(chatResponse(null, [toolCall('call_1', 'myTool')]))
+      .mockResolvedValueOnce(chatResponse('done'))
+
+    const result = await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'go' }],
+      tools: [makeTool('myTool')],
+    })
+
+    expect(result.toolCalls?.[0]).toMatchObject({ name: 'myTool', success: false })
+    const toolMessage = createPayload(1).messages.at(-1)
+    expect(JSON.parse(toolMessage.content)).toMatchObject({ error: true, tool: 'myTool' })
+  })
+
+  it('surfaces a ProviderError when a follow-up model call fails mid-loop', async () => {
+    mockCreate
+      .mockResolvedValueOnce(chatResponse(null, [toolCall('call_1', 'myTool')]))
+      .mockRejectedValueOnce(new Error('connection reset'))
+
+    await expect(
+      vllmProvider.executeRequest({
+        model: 'vllm/llama-3',
+        messages: [{ role: 'user', content: 'go' }],
+        tools: [makeTool('myTool')],
+      })
+    ).rejects.toThrow('connection reset')
+
+    expect(mockExecuteTool).toHaveBeenCalledTimes(1)
+  })
+
+  it('cycles forced tools: forces the next forced tool after the first is used', async () => {
+    mockPrepareTools.mockReturnValue({
+      tools: [{ type: 'function', function: { name: 'toolA' } }],
+      toolChoice: { type: 'function', function: { name: 'toolA' } },
+      forcedTools: ['toolA', 'toolB'],
+      hasFilteredTools: false,
+    })
+    mockCheckForced
+      .mockReturnValueOnce({ hasUsedForcedTool: true, usedForcedTools: ['toolA'] })
+      .mockReturnValueOnce({ hasUsedForcedTool: true, usedForcedTools: ['toolA', 'toolB'] })
+    mockCreate
+      .mockResolvedValueOnce(chatResponse(null, [toolCall('c1', 'toolA')]))
+      .mockResolvedValueOnce(chatResponse('done'))
+
+    await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'go' }],
+      tools: [makeTool('toolA'), makeTool('toolB')],
+    })
+
+    expect(createPayload(1).tool_choice).toEqual({ type: 'function', function: { name: 'toolB' } })
+  })
+
+  it('streams directly when there are no tools, requesting usage in the stream', async () => {
+    mockCreate.mockResolvedValueOnce({})
+
+    const result = await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'hi' }],
+      stream: true,
+    })
+
+    expect(mockCreate).toHaveBeenCalledTimes(1)
+    const payload = createPayload(0)
+    expect(payload.stream).toBe(true)
+    expect(payload.stream_options).toEqual({ include_usage: true })
+    expect('stream' in result && 'execution' in result).toBe(true)
+  })
+
+  it('uses tool_choice "none" on the final streaming call after tool processing', async () => {
+    mockCreate.mockResolvedValueOnce(chatResponse('answer')).mockResolvedValueOnce({})
+
+    await vllmProvider.executeRequest({
+      model: 'vllm/llama-3',
+      messages: [{ role: 'user', content: 'hi' }],
+      stream: true,
+      tools: [makeTool('myTool')],
+    })
+
+    const streamingPayload = createPayload(1)
+    expect(streamingPayload.stream).toBe(true)
+    expect(streamingPayload.tool_choice).toBe('none')
+  })
+
+  it('throws a ProviderError carrying the vLLM error message on API failure', async () => {
+    mockCreate.mockRejectedValueOnce({
+      error: { message: 'bad request', type: 'invalid', code: 400 },
+    })
+
+    await expect(
+      vllmProvider.executeRequest({
+        model: 'vllm/llama-3',
+        messages: [{ role: 'user', content: 'hi' }],
+      })
+    ).rejects.toThrow('bad request')
+  })
+
+  it('throws when no base URL is configured', async () => {
+    envState.VLLM_BASE_URL = ''
+
+    await expect(
+      vllmProvider.executeRequest({
+        model: 'vllm/llama-3',
+        messages: [{ role: 'user', content: 'hi' }],
+      })
+    ).rejects.toThrow('VLLM_BASE_URL is required')
+  })
+})

apps/sim/providers/vllm/index.ts

@@ -0,0 +1,670 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import OpenAI from 'openai'
+import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
+import { env } from '@/lib/core/config/env'
+import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server'
+import type { StreamingExecution } from '@/executor/types'
+import { MAX_TOOL_ITERATIONS } from '@/providers'
+import { formatMessagesForProvider } from '@/providers/attachments'
+import { getCachedProviderClient } from '@/providers/client-cache'
+import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
+import { createStreamingExecution } from '@/providers/streaming-execution'
+import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
+import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
+import type {
+  Message,
+  ProviderConfig,
+  ProviderRequest,
+  ProviderResponse,
+  TimeSegment,
+} from '@/providers/types'
+import { ProviderError } from '@/providers/types'
+import {
+  calculateCost,
+  prepareToolExecution,
+  prepareToolsWithUsageControl,
+  sumToolCosts,
+} from '@/providers/utils'
+import { checkForForcedToolUsage, createReadableStreamFromVLLMStream } from '@/providers/vllm/utils'
+import { useProvidersStore } from '@/stores/providers'
+import { executeTool } from '@/tools'
+
+const logger = createLogger('VLLMProvider')
+const VLLM_VERSION = '1.0.0'
+
+export const vllmProvider: ProviderConfig = {
+  id: 'vllm',
+  name: 'vLLM',
+  description: 'Self-hosted vLLM with OpenAI-compatible API',
+  version: VLLM_VERSION,
+  models: getProviderModels('vllm'),
+  defaultModel: getProviderDefaultModel('vllm'),
+
+  async initialize() {
+    if (typeof window !== 'undefined') {
+      logger.info('Skipping vLLM initialization on client side to avoid CORS issues')
+      return
+    }
+
+    const baseUrl = (env.VLLM_BASE_URL || '').replace(/\/$/, '')
+    if (!baseUrl) {
+      logger.info('VLLM_BASE_URL not configured, skipping initialization')
+      return
+    }
+
+    try {
+      const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+      }
+
+      if (env.VLLM_API_KEY) {
+        headers.Authorization = `Bearer ${env.VLLM_API_KEY}`
+      }
+
+      const response = await fetch(`${baseUrl}/v1/models`, { headers })
+      if (!response.ok) {
+        await response.text().catch(() => {})
+        useProvidersStore.getState().setProviderModels('vllm', [])
+        logger.warn('vLLM service is not available. The provider will be disabled.')
+        return
+      }
+
+      const data = (await response.json()) as { data: Array<{ id: string }> }
+      const models = data.data.map((model) => `vllm/${model.id}`)
+
+      this.models = models
+      useProvidersStore.getState().setProviderModels('vllm', models)
+
+      logger.info(`Discovered ${models.length} vLLM model(s):`, { models })
+    } catch (error) {
+      logger.warn('vLLM model instantiation failed. The provider will be disabled.', {
+        error: getErrorMessage(error, 'Unknown error'),
+      })
+    }
+  },
+
+  executeRequest: async (
+    request: ProviderRequest
+  ): Promise<ProviderResponse | StreamingExecution> => {
+    logger.info('Preparing vLLM request', {
+      model: request.model,
+      hasSystemPrompt: !!request.systemPrompt,
+      hasMessages: !!request.messages?.length,
+      hasTools: !!request.tools?.length,
+      toolCount: request.tools?.length || 0,
+      hasResponseFormat: !!request.responseFormat,
+      stream: !!request.stream,
+    })
+
+    const userProvidedEndpoint = request.azureEndpoint
+
+    const baseUrl = (userProvidedEndpoint || env.VLLM_BASE_URL || '').replace(/\/$/, '')
+    if (!baseUrl) {
+      throw new Error('VLLM_BASE_URL is required for vLLM provider')
+    }
+
+    /**
+     * A user-supplied endpoint is attacker-controlled: validate it against the
+     * central SSRF guard and pin the connection to the resolved IP to defeat DNS
+     * rebinding. The operator-configured `VLLM_BASE_URL` is trusted and left
+     * unvalidated, mirroring the Azure providers.
+     *
+     * `allowHttp` is enabled because self-hosted vLLM is frequently served over
+     * plain HTTP; this only relaxes the protocol requirement — the private/reserved
+     * IP blocklist and blocked-port checks still apply, so SSRF protection is intact.
+     */
+    let pinnedFetch: typeof fetch | undefined
+    let pinnedIP: string | undefined
+    if (userProvidedEndpoint) {
+      const validation = await validateUrlWithDNS(userProvidedEndpoint, 'vLLM endpoint', {
+        allowHttp: true,
+      })
+      if (!validation.isValid) {
+        logger.warn('Blocked SSRF attempt via vLLM endpoint', {
+          endpoint: userProvidedEndpoint,
+          error: validation.error,
+        })
+        throw new Error(`Invalid vLLM endpoint: ${validation.error}`)
+      }
+      if (!validation.resolvedIP) {
+        throw new Error('Invalid vLLM endpoint: could not resolve a pinnable IP address')
+      }
+      pinnedIP = validation.resolvedIP
+      pinnedFetch = createPinnedFetch(pinnedIP)
+    }
+
+    const apiKey = request.apiKey || env.VLLM_API_KEY || 'empty'
+    const vllm = getCachedProviderClient(
+      `vllm::${apiKey}::${baseUrl}::${pinnedIP ?? 'no-pin'}`,
+      () =>
+        new OpenAI({
+          apiKey,
+          baseURL: `${baseUrl}/v1`,
+          ...(pinnedFetch ? { fetch: pinnedFetch } : {}),
+        })
+    )
+
+    const allMessages: Message[] = []
+
+    if (request.systemPrompt) {
+      allMessages.push({
+        role: 'system',
+        content: request.systemPrompt,
+      })
+    }
+
+    if (request.context) {
+      allMessages.push({
+        role: 'user',
+        content: request.context,
+      })
+    }
+
+    if (request.messages) {
+      allMessages.push(...request.messages)
+    }
+    const formattedMessages = formatMessagesForProvider(allMessages, 'vllm') as Message[]
+
+    const tools = request.tools?.length
+      ? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool))
+      : undefined
+
+    const payload: any = {
+      model: request.model.replace(/^vllm\//, ''),
+      messages: formattedMessages,
+    }
+
+    if (request.temperature !== undefined) payload.temperature = request.temperature
+    if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens
+
+    if (request.responseFormat) {
+      payload.response_format = {
+        type: 'json_schema',
+        json_schema: {
+          name: request.responseFormat.name || 'response_schema',
+          schema: request.responseFormat.schema || request.responseFormat,
+          strict: request.responseFormat.strict !== false,
+        },
+      }
+
+      logger.info('Added JSON schema response format to vLLM request')
+    }
+
+    let preparedTools: ReturnType<typeof prepareToolsWithUsageControl> | null = null
+    let hasActiveTools = false
+
+    if (tools?.length) {
+      preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'vllm')
+      const { tools: filteredTools, toolChoice } = preparedTools
+
+      if (filteredTools?.length && toolChoice) {
+        payload.tools = filteredTools
+        payload.tool_choice = toolChoice
+        hasActiveTools = true
+
+        logger.info('vLLM request configuration:', {
+          toolCount: filteredTools.length,
+          toolChoice:
+            typeof toolChoice === 'string'
+              ? toolChoice
+              : toolChoice.type === 'function'
+                ? `force:${toolChoice.function.name}`
+                : 'unknown',
+          model: payload.model,
+        })
+      }
+    }
+
+    const providerStartTime = Date.now()
+    const providerStartTimeISO = new Date(providerStartTime).toISOString()
+
+    try {
+      if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) {
+        logger.info('Using streaming response for vLLM request')
+
+        const streamingParams: ChatCompletionCreateParamsStreaming = {
+          ...payload,
+          stream: true,
+          stream_options: { include_usage: true },
+        }
+        const streamResponse = await vllm.chat.completions.create(
+          streamingParams,
+          request.abortSignal ? { signal: request.abortSignal } : undefined
+        )
+
+        const streamingResult = createStreamingExecution({
+          model: request.model,
+          providerStartTime,
+          providerStartTimeISO,
+          timing: { kind: 'simple', segmentName: request.model },
+          initialTokens: { input: 0, output: 0, total: 0 },
+          initialCost: { input: 0, output: 0, total: 0 },
+          createStream: ({ output, finalizeTiming }) =>
+            createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
+              let cleanContent = content
+              if (cleanContent && request.responseFormat) {
+                cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
+              }
+
+              output.content = cleanContent
+              output.tokens = {
+                input: usage.prompt_tokens,
+                output: usage.completion_tokens,
+                total: usage.total_tokens,
+              }
+
+              const costResult = calculateCost(
+                request.model,
+                usage.prompt_tokens,
+                usage.completion_tokens
+              )
+              output.cost = {
+                input: costResult.input,
+                output: costResult.output,
+                total: costResult.total,
+              }
+
+              finalizeTiming()
+            }),
+        })
+
+        return streamingResult
+      }
+
+      const initialCallTime = Date.now()
+
+      const originalToolChoice = payload.tool_choice
+
+      const forcedTools = preparedTools?.forcedTools || []
+      let usedForcedTools: string[] = []
+      let hasUsedForcedTool = false
+
+      let currentResponse = await vllm.chat.completions.create(
+        payload,
+        request.abortSignal ? { signal: request.abortSignal } : undefined
+      )
+      const firstResponseTime = Date.now() - initialCallTime
+
+      let content = currentResponse.choices[0]?.message?.content || ''
+
+      if (content && request.responseFormat) {
+        content = content.replace(/```json\n?|\n?```/g, '').trim()
+      }
+
+      const tokens = {
+        input: currentResponse.usage?.prompt_tokens || 0,
+        output: currentResponse.usage?.completion_tokens || 0,
+        total: currentResponse.usage?.total_tokens || 0,
+      }
+      const toolCalls = []
+      const toolResults: Record<string, unknown>[] = []
+      const currentMessages = [...formattedMessages]
+      let iterationCount = 0
+
+      let modelTime = firstResponseTime
+      let toolsTime = 0
+
+      const timeSegments: TimeSegment[] = [
+        {
+          type: 'model',
+          name: request.model,
+          startTime: initialCallTime,
+          endTime: initialCallTime + firstResponseTime,
+          duration: firstResponseTime,
+        },
+      ]
+
+      if (originalToolChoice) {
+        const forcedResult = checkForForcedToolUsage(
+          currentResponse,
+          originalToolChoice,
+          forcedTools,
+          usedForcedTools
+        )
+        hasUsedForcedTool = forcedResult.hasUsedForcedTool
+        usedForcedTools = forcedResult.usedForcedTools
+      }
+
+      while (iterationCount < MAX_TOOL_ITERATIONS) {
+        if (currentResponse.choices[0]?.message?.content) {
+          content = currentResponse.choices[0].message.content
+          if (request.responseFormat) {
+            content = content.replace(/```json\n?|\n?```/g, '').trim()
+          }
+        }
+
+        const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls
+
+        enrichLastModelSegmentFromChatCompletions(
+          timeSegments,
+          currentResponse,
+          toolCallsInResponse,
+          { model: request.model, provider: 'vllm' }
+        )
+
+        if (!toolCallsInResponse || toolCallsInResponse.length === 0) {
+          break
+        }
+
+        logger.info(
+          `Processing ${toolCallsInResponse.length} tool calls (iteration ${iterationCount + 1}/${MAX_TOOL_ITERATIONS})`
+        )
+
+        const toolsStartTime = Date.now()
+
+        const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => {
+          const toolCallStartTime = Date.now()
+          const toolName = toolCall.function.name
+
+          try {
+            const toolArgs = JSON.parse(toolCall.function.arguments)
+            const tool = request.tools?.find((t) => t.id === toolName)
+
+            if (!tool) return null
+
+            const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
+            const result = await executeTool(toolName, executionParams, {
+              signal: request.abortSignal,
+            })
+            const toolCallEndTime = Date.now()
+
+            return {
+              toolCall,
+              toolName,
+              toolParams,
+              result,
+              startTime: toolCallStartTime,
+              endTime: toolCallEndTime,
+              duration: toolCallEndTime - toolCallStartTime,
+            }
+          } catch (error) {
+            const toolCallEndTime = Date.now()
+            logger.error('Error processing tool call:', { error, toolName })
+
+            return {
+              toolCall,
+              toolName,
+              toolParams: {},
+              result: {
+                success: false,
+                output: undefined,
+                error: getErrorMessage(error, 'Tool execution failed'),
+              },
+              startTime: toolCallStartTime,
+              endTime: toolCallEndTime,
+              duration: toolCallEndTime - toolCallStartTime,
+            }
+          }
+        })
+
+        const executionResults = await Promise.allSettled(toolExecutionPromises)
+
+        currentMessages.push({
+          role: 'assistant',
+          content: null,
+          tool_calls: toolCallsInResponse.map((tc) => ({
+            id: tc.id,
+            type: 'function',
+            function: {
+              name: tc.function.name,
+              arguments: tc.function.arguments,
+            },
+          })),
+        })
+
+        for (const settledResult of executionResults) {
+          if (settledResult.status === 'rejected' || !settledResult.value) continue
+
+          const { toolCall, toolName, toolParams, result, startTime, endTime, duration } =
+            settledResult.value
+
+          timeSegments.push({
+            type: 'tool',
+            name: toolName,
+            startTime: startTime,
+            endTime: endTime,
+            duration: duration,
+            toolCallId: toolCall.id,
+          })
+
+          let resultContent: any
+          if (result.success && result.output) {
+            toolResults.push(result.output)
+            resultContent = result.output
+          } else {
+            resultContent = {
+              error: true,
+              message: result.error || 'Tool execution failed',
+              tool: toolName,
+            }
+          }
+
+          toolCalls.push({
+            name: toolName,
+            arguments: toolParams,
+            startTime: new Date(startTime).toISOString(),
+            endTime: new Date(endTime).toISOString(),
+            duration: duration,
+            result: resultContent,
+            success: result.success,
+          })
+
+          currentMessages.push({
+            role: 'tool',
+            tool_call_id: toolCall.id,
+            content: JSON.stringify(resultContent),
+          })
+        }
+
+        const thisToolsTime = Date.now() - toolsStartTime
+        toolsTime += thisToolsTime
+
+        const nextPayload = {
+          ...payload,
+          messages: currentMessages,
+        }
+
+        if (typeof originalToolChoice === 'object' && hasUsedForcedTool && forcedTools.length > 0) {
+          const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool))
+
+          if (remainingTools.length > 0) {
+            nextPayload.tool_choice = {
+              type: 'function',
+              function: { name: remainingTools[0] },
+            }
+            logger.info(`Forcing next tool: ${remainingTools[0]}`)
+          } else {
+            nextPayload.tool_choice = 'auto'
+            logger.info('All forced tools have been used, switching to auto tool_choice')
+          }
+        }
+
+        const nextModelStartTime = Date.now()
+
+        currentResponse = await vllm.chat.completions.create(
+          nextPayload,
+          request.abortSignal ? { signal: request.abortSignal } : undefined
+        )
+
+        if (nextPayload.tool_choice && typeof nextPayload.tool_choice === 'object') {
+          const forcedResult = checkForForcedToolUsage(
+            currentResponse,
+            nextPayload.tool_choice,
+            forcedTools,
+            usedForcedTools
+          )
+          hasUsedForcedTool = forcedResult.hasUsedForcedTool
+          usedForcedTools = forcedResult.usedForcedTools
+        }
+
+        const nextModelEndTime = Date.now()
+        const thisModelTime = nextModelEndTime - nextModelStartTime
+
+        timeSegments.push({
+          type: 'model',
+          name: request.model,
+          startTime: nextModelStartTime,
+          endTime: nextModelEndTime,
+          duration: thisModelTime,
+        })
+
+        modelTime += thisModelTime
+
+        if (currentResponse.choices[0]?.message?.content) {
+          content = currentResponse.choices[0].message.content
+          if (request.responseFormat) {
+            content = content.replace(/```json\n?|\n?```/g, '').trim()
+          }
+        }
+
+        if (currentResponse.usage) {
+          tokens.input += currentResponse.usage.prompt_tokens || 0
+          tokens.output += currentResponse.usage.completion_tokens || 0
+          tokens.total += currentResponse.usage.total_tokens || 0
+        }
+
+        iterationCount++
+      }
+
+      if (iterationCount === MAX_TOOL_ITERATIONS) {
+        enrichLastModelSegmentFromChatCompletions(
+          timeSegments,
+          currentResponse,
+          currentResponse.choices[0]?.message?.tool_calls,
+          { model: request.model, provider: 'vllm' }
+        )
+      }
+
+      if (request.stream) {
+        logger.info('Using streaming for final response after tool processing')
+
+        const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output)
+
+        const streamingParams: ChatCompletionCreateParamsStreaming = {
+          ...payload,
+          messages: currentMessages,
+          tool_choice: 'none',
+          stream: true,
+          stream_options: { include_usage: true },
+        }
+        const streamResponse = await vllm.chat.completions.create(
+          streamingParams,
+          request.abortSignal ? { signal: request.abortSignal } : undefined
+        )
+
+        const streamingResult = createStreamingExecution({
+          model: request.model,
+          providerStartTime,
+          providerStartTimeISO,
+          timing: {
+            kind: 'accumulated',
+            modelTime,
+            toolsTime,
+            firstResponseTime,
+            iterations: iterationCount + 1,
+            timeSegments,
+          },
+          initialTokens: {
+            input: tokens.input,
+            output: tokens.output,
+            total: tokens.total,
+          },
+          initialCost: {
+            input: accumulatedCost.input,
+            output: accumulatedCost.output,
+            total: accumulatedCost.total,
+          },
+          toolCalls:
+            toolCalls.length > 0
+              ? {
+                  list: toolCalls,
+                  count: toolCalls.length,
+                }
+              : undefined,
+          createStream: ({ output }) =>
+            createReadableStreamFromVLLMStream(streamResponse, (content, usage) => {
+              let cleanContent = content
+              if (cleanContent && request.responseFormat) {
+                cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim()
+              }
+
+              output.content = cleanContent
+              output.tokens = {
+                input: tokens.input + usage.prompt_tokens,
+                output: tokens.output + usage.completion_tokens,
+                total: tokens.total + usage.total_tokens,
+              }
+
+              const streamCost = calculateCost(
+                request.model,
+                usage.prompt_tokens,
+                usage.completion_tokens
+              )
+              const tc = sumToolCosts(toolResults)
+              output.cost = {
+                input: accumulatedCost.input + streamCost.input,
+                output: accumulatedCost.output + streamCost.output,
+                toolCost: tc || undefined,
+                total: accumulatedCost.total + streamCost.total + tc,
+              }
+            }),
+        })
+
+        return streamingResult
+      }
+
+      const providerEndTime = Date.now()
+      const providerEndTimeISO = new Date(providerEndTime).toISOString()
+      const totalDuration = providerEndTime - providerStartTime
+
+      return {
+        content,
+        model: request.model,
+        tokens,
+        toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
+        toolResults: toolResults.length > 0 ? toolResults : undefined,
+        timing: {
+          startTime: providerStartTimeISO,
+          endTime: providerEndTimeISO,
+          duration: totalDuration,
+          modelTime: modelTime,
+          toolsTime: toolsTime,
+          firstResponseTime: firstResponseTime,
+          iterations: iterationCount + 1,
+          timeSegments: timeSegments,
+        },
+      }
+    } catch (error) {
+      const providerEndTime = Date.now()
+      const providerEndTimeISO = new Date(providerEndTime).toISOString()
+      const totalDuration = providerEndTime - providerStartTime
+
+      let errorMessage = toError(error).message
+      let errorType: string | undefined
+      let errorCode: number | undefined
+
+      if (error && typeof error === 'object' && 'error' in error) {
+        const vllmError = error.error as any
+        if (vllmError && typeof vllmError === 'object') {
+          errorMessage = vllmError.message || errorMessage
+          errorType = vllmError.type
+          errorCode = vllmError.code
+        }
+      }
+
+      logger.error('Error in vLLM request:', {
+        error: errorMessage,
+        errorType,
+        errorCode,
+        duration: totalDuration,
+      })
+
+      throw new ProviderError(errorMessage, {
+        startTime: providerStartTimeISO,
+        endTime: providerEndTimeISO,
+        duration: totalDuration,
+      })
+    }
+  },
+}

apps/sim/providers/vllm/utils.ts

@@ -0,0 +1,27 @@
+import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
+import type { CompletionUsage } from 'openai/resources/completions'
+import { checkForForcedToolUsageOpenAI, createOpenAICompatibleStream } from '@/providers/utils'
+
+/**
+ * Creates a ReadableStream from a vLLM streaming response.
+ * Uses the shared OpenAI-compatible streaming utility.
+ */
+export function createReadableStreamFromVLLMStream(
+  vllmStream: AsyncIterable<ChatCompletionChunk>,
+  onComplete?: (content: string, usage: CompletionUsage) => void
+): ReadableStream<Uint8Array> {
+  return createOpenAICompatibleStream(vllmStream, 'vLLM', onComplete)
+}
+
+/**
+ * Checks if a forced tool was used in a vLLM response.
+ * Uses the shared OpenAI-compatible forced tool usage helper.
+ */
+export function checkForForcedToolUsage(
+  response: any,
+  toolChoice: string | { type: string; function?: { name: string }; name?: string; any?: any },
+  forcedTools: string[],
+  usedForcedTools: string[]
+): { hasUsedForcedTool: boolean; usedForcedTools: string[] } {
+  return checkForForcedToolUsageOpenAI(response, toolChoice, 'vLLM', forcedTools, usedForcedTools)
+}