chore: import upstream snapshot with attribution

test/providers/openai/agents-tracing.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from 'vitest';
+import { OTLPTracingExporter } from '../../../src/providers/openai/agents-tracing';
+
+function getAttributes(span: any): Record<string, unknown> {
+  return Object.fromEntries(
+    span.attributes.map((attribute: any) => {
+      const value = attribute.value;
+      if (value.stringValue !== undefined) {
+        return [attribute.key, value.stringValue];
+      }
+      if (value.intValue !== undefined) {
+        return [attribute.key, Number(value.intValue)];
+      }
+      if (value.boolValue !== undefined) {
+        return [attribute.key, value.boolValue];
+      }
+      if (value.doubleValue !== undefined) {
+        return [attribute.key, value.doubleValue];
+      }
+      return [attribute.key, value];
+    }),
+  );
+}
+
+describe('OTLPTracingExporter', () => {
+  it('maps function spans into Promptfoo trajectory-friendly tool attributes', () => {
+    const exporter = new OTLPTracingExporter() as any;
+    const payload = exporter.transformToOTLP([
+      {
+        type: 'trace.span',
+        traceId: 'trace_0123456789abcdef0123456789abcdef',
+        spanId: 'span_0123456789abcdef',
+        parentId: null,
+        startedAt: '2026-05-06T12:00:00.000Z',
+        endedAt: '2026-05-06T12:00:01.000Z',
+        spanData: {
+          type: 'function',
+          name: 'lookup_order',
+          input: '{"order_id":"123"}',
+          output: '{"status":"shipped"}',
+        },
+        traceMetadata: {
+          'evaluation.id': 'eval-1',
+          'test.case.id': 'case-1',
+          'promptfoo.parent_span_id': 'fedcba9876543210',
+        },
+        error: null,
+      },
+    ]);
+
+    const span = payload.resourceSpans[0].scopeSpans[0].spans[0];
+    expect(span.name).toBe('tool lookup_order');
+    expect(getAttributes(span)).toMatchObject({
+      'evaluation.id': 'eval-1',
+      'openai.agents.span_type': 'function',
+      'test.case.id': 'case-1',
+      'tool.arguments': '{"order_id":"123"}',
+      'tool.name': 'lookup_order',
+      'tool.output': '{"status":"shipped"}',
+    });
+    expect(span.parentSpanId).toBe(Buffer.from('fedcba9876543210', 'hex').toString('base64'));
+  });
+
+  it('turns sandbox custom spans into command-aware spans', () => {
+    const exporter = new OTLPTracingExporter() as any;
+    const payload = exporter.transformToOTLP([
+      {
+        type: 'trace.span',
+        traceId: 'trace_0123456789abcdef0123456789abcdef',
+        spanId: 'span_0123456789abcdef',
+        parentId: null,
+        startedAt: '2026-05-06T12:00:00.000Z',
+        endedAt: '2026-05-06T12:00:01.000Z',
+        spanData: {
+          type: 'custom',
+          name: 'sandbox.exec',
+          data: {
+            cmd: ['cat', 'repo/task.md'],
+            workdir: 'repo',
+          },
+        },
+        traceMetadata: {},
+        error: null,
+      },
+    ]);
+
+    const span = payload.resourceSpans[0].scopeSpans[0].spans[0];
+    expect(span.name).toBe('sandbox.exec');
+    expect(getAttributes(span)).toMatchObject({
+      command: 'cat repo/task.md',
+      cmd: {
+        arrayValue: {
+          values: [{ stringValue: 'cat' }, { stringValue: 'repo/task.md' }],
+        },
+      },
+      'openai.agents.custom_span.name': 'sandbox.exec',
+      'openai.agents.span_type': 'custom',
+      workdir: 'repo',
+    });
+  });
+});

test/providers/openai/assistant.test.ts

@@ -0,0 +1,841 @@
+import { trace } from '@opentelemetry/api';
+import OpenAI from 'openai';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { disableCache, enableCache } from '../../../src/cache';
+import { OpenAiAssistantProvider } from '../../../src/providers/openai/assistant';
+import { mockProcessEnv } from '../../util/utils';
+import { getOpenAiMissingApiKeyMessage } from './shared';
+
+import type { CallbackContext } from '../../../src/providers/openai/types';
+
+vi.mock('openai');
+
+interface RecordedSpan {
+  name: string;
+  attributes: Record<string, any>;
+  status?: { code: number; message?: string };
+  ended: boolean;
+}
+
+function installTracerSpy(): RecordedSpan[] {
+  const spans: RecordedSpan[] = [];
+  const make = (name: string, attributes: Record<string, any> = {}) => {
+    const entry: RecordedSpan = { name, attributes: { ...attributes }, ended: false };
+    spans.push(entry);
+    return {
+      setAttribute: (key: string, value: unknown) => {
+        entry.attributes[key] = value;
+      },
+      setAttributes: (attrs: Record<string, unknown>) => Object.assign(entry.attributes, attrs),
+      setStatus: (status: { code: number; message?: string }) => {
+        entry.status = status;
+      },
+      end: () => {
+        entry.ended = true;
+      },
+      recordException: () => undefined,
+      addEvent: () => undefined,
+      spanContext: () => ({ traceId: 'x', spanId: 'y' }),
+      isRecording: () => true,
+      updateName: () => undefined,
+    };
+  };
+  vi.spyOn(trace, 'getTracer').mockReturnValue({
+    startSpan: (name: string, options?: { attributes?: Record<string, unknown> }) =>
+      make(name, options?.attributes),
+    startActiveSpan: (...args: any[]) => {
+      const name = args[0];
+      const options = typeof args[1] === 'object' ? args[1] : undefined;
+      const callback = args[args.length - 1];
+      return callback(make(name, options?.attributes));
+    },
+  } as any);
+  return spans;
+}
+
+describe('OpenAI Provider', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    disableCache();
+  });
+
+  afterEach(() => {
+    enableCache();
+  });
+
+  describe('OpenAiAssistantProvider', () => {
+    let mockClient: any;
+
+    beforeEach(() => {
+      vi.clearAllMocks();
+      mockClient = {
+        beta: {
+          threads: {
+            createAndRun: vi.fn(),
+            runs: {
+              retrieve: vi.fn(),
+              submitToolOutputs: vi.fn(),
+              steps: {
+                list: vi.fn(),
+              },
+            },
+            messages: {
+              retrieve: vi.fn(),
+            },
+          },
+        },
+      };
+      vi.mocked(OpenAI).mockImplementation(function (this: any) {
+        Object.assign(this, mockClient);
+        return this;
+      });
+    });
+
+    const provider = new OpenAiAssistantProvider('test-assistant-id', {
+      config: {
+        apiKey: 'test-key',
+        organization: 'test-org',
+        functionToolCallbacks: {
+          test_function: async (_args: string) => 'Function result',
+        },
+      },
+    });
+
+    it('should handle successful assistant completion', async () => {
+      const mockRun = {
+        id: 'run_123',
+        thread_id: 'thread_123',
+        status: 'completed',
+      };
+
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_1',
+            step_details: {
+              type: 'message_creation',
+              message_creation: {
+                message_id: 'msg_1',
+              },
+            },
+          },
+        ],
+      };
+
+      const mockMessage = {
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'Test response',
+            },
+          },
+        ],
+      };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+      mockClient.beta.threads.messages.retrieve.mockResolvedValue(mockMessage);
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(OpenAI).toHaveBeenCalledWith(
+        expect.objectContaining({
+          defaultHeaders: expect.objectContaining({
+            'X-OpenAI-Originator': 'promptfoo',
+          }),
+        }),
+      );
+      expect(result.output).toBe('[Assistant] Test response');
+      expect(mockClient.beta.threads.createAndRun).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.runs.retrieve).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.runs.steps.list).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.messages.retrieve).toHaveBeenCalledTimes(1);
+    });
+
+    it('drops the SDK organization option when a case-variant org header overrides it', async () => {
+      const mockRun = { id: 'run_123', thread_id: 'thread_123', status: 'completed' };
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_1',
+            step_details: { type: 'message_creation', message_creation: { message_id: 'msg_1' } },
+          },
+        ],
+      };
+      const mockMessage = {
+        role: 'assistant',
+        content: [{ type: 'text', text: { value: 'Test response' } }],
+      };
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+      mockClient.beta.threads.messages.retrieve.mockResolvedValue(mockMessage);
+
+      const overrideProvider = new OpenAiAssistantProvider('test-assistant-id', {
+        config: {
+          apiKey: 'test-key',
+          organization: 'test-org',
+          headers: { 'openai-organization': 'custom-org' },
+        },
+      });
+
+      await overrideProvider.callApi('Test prompt');
+
+      // The SDK would otherwise inject its own canonical OpenAI-Organization header
+      // from `organization`, duplicating (and beating) the custom override.
+      expect(OpenAI).toHaveBeenCalledWith(
+        expect.objectContaining({
+          organization: undefined,
+          defaultHeaders: expect.objectContaining({ 'openai-organization': 'custom-org' }),
+        }),
+      );
+      const sdkArgs = vi.mocked(OpenAI).mock.calls.at(-1)?.[0] as { defaultHeaders: object };
+      expect(sdkArgs.defaultHeaders).not.toHaveProperty('OpenAI-Organization');
+    });
+
+    it('emits a chat <model> span around the assistant run', async () => {
+      const spans = installTracerSpy();
+      const mockRun = { id: 'run_123', thread_id: 'thread_123', status: 'completed' };
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_1',
+            step_details: { type: 'message_creation', message_creation: { message_id: 'msg_1' } },
+          },
+        ],
+      };
+      const mockMessage = {
+        role: 'assistant',
+        content: [{ type: 'text', text: { value: 'Test response' } }],
+      };
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+      mockClient.beta.threads.messages.retrieve.mockResolvedValue(mockMessage);
+
+      const localProvider = new OpenAiAssistantProvider('test-assistant-id', {
+        config: { apiKey: 'test-key' },
+      });
+      await localProvider.callApi('Test prompt');
+
+      const chatSpan = spans.find((span) => span.name === 'chat test-assistant-id');
+      expect(chatSpan).toBeDefined();
+      expect(chatSpan?.attributes).toMatchObject({
+        'gen_ai.system': 'openai',
+        'gen_ai.operation.name': 'chat',
+        'gen_ai.request.model': 'test-assistant-id',
+      });
+      expect(chatSpan?.ended).toBe(true);
+      // SpanStatusCode.OK === 1
+      expect(chatSpan?.status?.code).toBe(1);
+    });
+
+    it('marks the chat span ERROR when the assistant run fails', async () => {
+      const spans = installTracerSpy();
+      mockClient.beta.threads.createAndRun.mockRejectedValue(new Error('assistant boom'));
+
+      const localProvider = new OpenAiAssistantProvider('test-assistant-id', {
+        config: { apiKey: 'test-key' },
+      });
+      const result = await localProvider.callApi('Test prompt');
+      expect(result.error).toBeDefined();
+
+      const chatSpan = spans.find((span) => span.name === 'chat test-assistant-id');
+      expect(chatSpan).toBeDefined();
+      // SpanStatusCode.ERROR === 2
+      expect(chatSpan?.status?.code).toBe(2);
+      expect(chatSpan?.ended).toBe(true);
+    });
+
+    it('should preserve an explicit temperature of 0', async () => {
+      const mockRun = {
+        id: 'run_123',
+        thread_id: 'thread_123',
+        status: 'completed',
+      };
+
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_1',
+            step_details: {
+              type: 'message_creation',
+              message_creation: {
+                message_id: 'msg_1',
+              },
+            },
+          },
+        ],
+      };
+
+      const mockMessage = {
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'Test response',
+            },
+          },
+        ],
+      };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+      mockClient.beta.threads.messages.retrieve.mockResolvedValue(mockMessage);
+
+      const provider = new OpenAiAssistantProvider('test-assistant-id', {
+        config: {
+          apiKey: 'test-key',
+          temperature: 0,
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      expect(mockClient.beta.threads.createAndRun).toHaveBeenCalledWith(
+        expect.objectContaining({
+          temperature: 0,
+        }),
+      );
+    });
+
+    it('should handle function calling', async () => {
+      const mockRun = {
+        id: 'run_123',
+        thread_id: 'thread_123',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_123',
+                type: 'function',
+                function: {
+                  name: 'test_function',
+                  arguments: '{"arg": "value"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = {
+        ...mockRun,
+        status: 'completed',
+        required_action: null,
+      };
+
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_1',
+            step_details: {
+              type: 'tool_calls',
+              tool_calls: [
+                {
+                  type: 'function',
+                  function: {
+                    name: 'test_function',
+                    arguments: '{"arg": "value"}',
+                    output: 'Function result',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toBe(
+        '[Call function test_function with arguments {"arg": "value"}]\n\n[Function output: Function result]',
+      );
+      expect(mockClient.beta.threads.createAndRun).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.runs.retrieve).toHaveBeenCalledTimes(2);
+      expect(mockClient.beta.threads.runs.submitToolOutputs).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.runs.steps.list).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle run failures', async () => {
+      const mockRun = {
+        id: 'run_123',
+        thread_id: 'thread_123',
+        status: 'failed',
+        last_error: {
+          message: 'Test error message',
+        },
+      };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve.mockResolvedValue(mockRun);
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.error).toBe('Thread run failed: Test error message');
+      expect(mockClient.beta.threads.createAndRun).toHaveBeenCalledTimes(1);
+      expect(mockClient.beta.threads.runs.retrieve).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle API errors', async () => {
+      const error = new OpenAI.APIError(500, {}, 'API Error', new Headers());
+      Object.defineProperty(error, 'type', {
+        value: 'API Error',
+        writable: true,
+        configurable: true,
+      });
+      Object.defineProperty(error, 'message', {
+        value: 'API Error',
+        writable: true,
+        configurable: true,
+      });
+
+      mockClient.beta.threads.createAndRun.mockRejectedValueOnce(error);
+
+      const provider = new OpenAiAssistantProvider('test-assistant-id', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.error).toBe('API error: API Error API Error');
+      expect(mockClient.beta.threads.createAndRun).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle missing API key', async () => {
+      const restoreEnv = mockProcessEnv({ OPENAI_API_KEY: undefined });
+
+      try {
+        const providerNoKey = new OpenAiAssistantProvider('test-assistant-id', {
+          env: {
+            OPENAI_API_KEY: undefined,
+          },
+        });
+
+        await expect(providerNoKey.callApi('Test prompt')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage(),
+        );
+      } finally {
+        restoreEnv();
+      }
+    });
+
+    it('should use custom apiKeyEnvar in missing API key errors', async () => {
+      const restoreEnv = mockProcessEnv({
+        OPENAI_API_KEY: undefined,
+        CUSTOM_ASSISTANT_API_KEY: undefined,
+      });
+
+      try {
+        const providerNoKey = new OpenAiAssistantProvider('test-assistant-id', {
+          config: {
+            apiKeyEnvar: 'CUSTOM_ASSISTANT_API_KEY',
+          },
+          env: {
+            OPENAI_API_KEY: undefined,
+            CUSTOM_ASSISTANT_API_KEY: undefined,
+          },
+        });
+
+        await expect(providerNoKey.callApi('Test prompt')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage('CUSTOM_ASSISTANT_API_KEY'),
+        );
+      } finally {
+        restoreEnv();
+      }
+    });
+  });
+
+  describe('Function Callbacks with Context', () => {
+    let mockClient: any;
+
+    beforeEach(() => {
+      vi.clearAllMocks();
+      disableCache();
+
+      mockClient = {
+        beta: {
+          threads: {
+            createAndRun: vi.fn(),
+            runs: {
+              retrieve: vi.fn(),
+              submitToolOutputs: vi.fn(),
+              steps: {
+                list: vi.fn(),
+              },
+            },
+            messages: {
+              retrieve: vi.fn(),
+            },
+          },
+        },
+      };
+
+      vi.mocked(OpenAI).mockImplementation(function (this: any) {
+        Object.assign(this, mockClient);
+        return this;
+      });
+    });
+
+    it('should pass context to function callbacks', async () => {
+      const mockCallback = vi.fn().mockResolvedValue('test result');
+
+      const provider = new OpenAiAssistantProvider('asst_test', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            test_function: mockCallback,
+          },
+        },
+      });
+
+      const mockRun = {
+        id: 'run_test',
+        thread_id: 'thread_test',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_test',
+                type: 'function',
+                function: {
+                  name: 'test_function',
+                  arguments: '{"param": "value"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = {
+        ...mockRun,
+        status: 'completed',
+      };
+
+      const mockSteps = {
+        data: [
+          {
+            id: 'step_test',
+            step_details: {
+              type: 'message_creation',
+              message_creation: {
+                message_id: 'msg_test',
+              },
+            },
+          },
+        ],
+      };
+
+      const mockMessage = {
+        id: 'msg_test',
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'Test response',
+            },
+          },
+        ],
+      };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+      mockClient.beta.threads.messages.retrieve.mockResolvedValue(mockMessage);
+
+      await provider.callApi('test prompt');
+
+      // Verify that the callback was called with the correct context
+      expect(mockCallback).toHaveBeenCalledWith(
+        { param: 'value' },
+        {
+          threadId: 'thread_test',
+          runId: 'run_test',
+          assistantId: 'asst_test',
+          provider: 'openai',
+        },
+      );
+    });
+
+    it('should work with callbacks that do not use context', async () => {
+      const oldStyleCallback = vi.fn().mockResolvedValue('old style result');
+
+      const provider = new OpenAiAssistantProvider('asst_test', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            old_function: oldStyleCallback,
+          },
+        },
+      });
+
+      const mockRun = {
+        id: 'run_test',
+        thread_id: 'thread_test',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_test',
+                type: 'function',
+                function: {
+                  name: 'old_function',
+                  arguments: '{"param": "value"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = { ...mockRun, status: 'completed' };
+      const mockSteps = { data: [] };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+
+      await provider.callApi('test prompt');
+
+      // Callback should be called with args and context, but context is optional
+      expect(oldStyleCallback).toHaveBeenCalledWith(
+        { param: 'value' },
+        expect.objectContaining({
+          threadId: 'thread_test',
+          runId: 'run_test',
+          assistantId: 'asst_test',
+          provider: 'openai',
+        }),
+      );
+    });
+
+    it('should handle string-based function callbacks', async () => {
+      const provider = new OpenAiAssistantProvider('asst_test', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            string_function:
+              '(args, context) => { return `received: ${JSON.stringify(args)} with context: ${JSON.stringify(context)}`; }',
+          },
+        },
+      });
+
+      const mockRun = {
+        id: 'run_test',
+        thread_id: 'thread_test',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_test',
+                type: 'function',
+                function: {
+                  name: 'string_function',
+                  arguments: '{"test": "data"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = { ...mockRun, status: 'completed' };
+      const mockSteps = { data: [] };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+
+      await provider.callApi('test prompt');
+
+      // Check that the tool output was submitted correctly
+      expect(mockClient.beta.threads.runs.submitToolOutputs).toHaveBeenCalledWith('run_test', {
+        thread_id: 'thread_test',
+        tool_outputs: [
+          {
+            tool_call_id: 'call_test',
+            output: expect.stringContaining('received: {"test":"data"}'),
+          },
+        ],
+      });
+    });
+
+    it('should handle callbacks that access context properties', async () => {
+      const contextAwareCallback = vi.fn().mockImplementation(function (
+        args: any,
+        context?: CallbackContext,
+      ) {
+        const result = {
+          originalArgs: args,
+          contextInfo: {
+            threadId: context?.threadId,
+            provider: context?.provider,
+          },
+        };
+        return Promise.resolve(JSON.stringify(result));
+      });
+
+      const provider = new OpenAiAssistantProvider('asst_test', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            context_function: contextAwareCallback,
+          },
+        },
+      });
+
+      const mockRun = {
+        id: 'run_test',
+        thread_id: 'thread_test',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_test',
+                type: 'function',
+                function: {
+                  name: 'context_function',
+                  arguments: '{"user_id": "123"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = { ...mockRun, status: 'completed' };
+      const mockSteps = { data: [] };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+
+      await provider.callApi('test prompt');
+
+      // Verify the callback was called with the correct parameters
+      expect(contextAwareCallback).toHaveBeenCalledWith(
+        { user_id: '123' },
+        expect.objectContaining({
+          threadId: 'thread_test',
+          runId: 'run_test',
+          assistantId: 'asst_test',
+          provider: 'openai',
+        }),
+      );
+
+      // Verify the tool output contains the context information
+      expect(mockClient.beta.threads.runs.submitToolOutputs).toHaveBeenCalledWith('run_test', {
+        thread_id: 'thread_test',
+        tool_outputs: [
+          {
+            tool_call_id: 'call_test',
+            output: JSON.stringify({
+              originalArgs: { user_id: '123' },
+              contextInfo: {
+                threadId: 'thread_test',
+                provider: 'openai',
+              },
+            }),
+          },
+        ],
+      });
+    });
+
+    it('should handle function callback errors gracefully', async () => {
+      const errorCallback = vi.fn().mockRejectedValue(new Error('Callback error'));
+
+      const provider = new OpenAiAssistantProvider('asst_test', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            error_function: errorCallback,
+          },
+        },
+      });
+
+      const mockRun = {
+        id: 'run_test',
+        thread_id: 'thread_test',
+        status: 'requires_action',
+        required_action: {
+          type: 'submit_tool_outputs',
+          submit_tool_outputs: {
+            tool_calls: [
+              {
+                id: 'call_test',
+                type: 'function',
+                function: {
+                  name: 'error_function',
+                  arguments: '{"param": "value"}',
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const mockCompletedRun = { ...mockRun, status: 'completed' };
+      const mockSteps = { data: [] };
+
+      mockClient.beta.threads.createAndRun.mockResolvedValue(mockRun);
+      mockClient.beta.threads.runs.retrieve
+        .mockResolvedValueOnce(mockRun)
+        .mockResolvedValueOnce(mockCompletedRun);
+      mockClient.beta.threads.runs.submitToolOutputs.mockResolvedValue(mockCompletedRun);
+      mockClient.beta.threads.runs.steps.list.mockResolvedValue(mockSteps);
+
+      await provider.callApi('test prompt');
+
+      // Verify error was handled and submitted as tool output
+      expect(mockClient.beta.threads.runs.submitToolOutputs).toHaveBeenCalledWith('run_test', {
+        thread_id: 'thread_test',
+        tool_outputs: [
+          {
+            tool_call_id: 'call_test',
+            output: JSON.stringify({
+              error: 'Error in error_function: Callback error',
+            }),
+          },
+        ],
+      });
+    });
+  });
+});

test/providers/openai/billing.test.ts

@@ -0,0 +1,737 @@
+import { describe, expect, it } from 'vitest';
+import {
+  calculateObservableOpenAIToolCost,
+  calculateOpenAIUsageCost,
+  calculateOpenAIUsageCostFromTokenUsage,
+  extractOpenAIBillingUsage,
+} from '../../../src/providers/openai/billing';
+
+describe('OpenAI billing helpers', () => {
+  it('extracts multimodal usage details from responses payloads', () => {
+    expect(
+      extractOpenAIBillingUsage({
+        input_tokens: 220,
+        output_tokens: 460,
+        input_tokens_details: {
+          text_tokens: 20,
+          image_tokens: 194,
+          audio_tokens: 6,
+          cached_tokens: 12,
+          cache_write_tokens: 5,
+        },
+        output_tokens_details: {
+          text_tokens: 180,
+          image_tokens: 272,
+          audio_tokens: 8,
+        },
+      }),
+    ).toEqual({
+      totalInputTokens: 220,
+      cachedInputTokens: 12,
+      cacheWriteInputTokens: 5,
+      cachedTextInputTokens: 0,
+      cachedAudioInputTokens: 0,
+      cachedImageInputTokens: 0,
+      textInputTokens: 20,
+      audioInputTokens: 6,
+      imageInputTokens: 194,
+      totalOutputTokens: 460,
+      textOutputTokens: 180,
+      audioOutputTokens: 8,
+      imageOutputTokens: 272,
+    });
+  });
+
+  it('extracts multimodal usage details from realtime payloads', () => {
+    expect(
+      extractOpenAIBillingUsage({
+        input_tokens: 30,
+        output_tokens: 23,
+        input_token_details: {
+          text_tokens: 21,
+          audio_tokens: 9,
+          cached_tokens: 4,
+          cached_tokens_details: {
+            text_tokens: 3,
+            audio_tokens: 1,
+            image_tokens: 0,
+          },
+        },
+        output_token_details: {
+          text_tokens: 16,
+          audio_tokens: 7,
+        },
+      }),
+    ).toEqual({
+      totalInputTokens: 30,
+      cachedInputTokens: 4,
+      cacheWriteInputTokens: 0,
+      cachedTextInputTokens: 3,
+      cachedAudioInputTokens: 1,
+      cachedImageInputTokens: 0,
+      textInputTokens: 21,
+      audioInputTokens: 9,
+      imageInputTokens: 0,
+      totalOutputTokens: 23,
+      textOutputTokens: 16,
+      audioOutputTokens: 7,
+      imageOutputTokens: 0,
+    });
+  });
+
+  it('applies provider-side cached-input discounts to standard text usage', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-5-mini',
+      {},
+      {
+        prompt_tokens: 1_000,
+        completion_tokens: 100,
+        prompt_tokens_details: { cached_tokens: 400 },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo((600 * 0.25 + 400 * 0.025 + 100 * 2) / 1e6, 10);
+  });
+
+  it('prices cached input for GPT-5.3 coding models', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.3-codex',
+        {},
+        {
+          prompt_tokens: 2_000,
+          completion_tokens: 1_000,
+          prompt_tokens_details: { cached_tokens: 500 },
+        },
+      ),
+    ).toBeCloseTo((1_500 * 1.75 + 500 * 0.175 + 1_000 * 14) / 1e6, 10);
+
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.3-codex-spark',
+        {},
+        {
+          prompt_tokens: 2_000,
+          completion_tokens: 1_000,
+          prompt_tokens_details: { cached_tokens: 500 },
+        },
+      ),
+    ).toBeCloseTo((1_500 * 0.5 + 500 * 0.05 + 1_000 * 4) / 1e6, 10);
+  });
+
+  it.each([
+    ['gpt-5.6', 5, 0.5, 30],
+    ['gpt-5.6-sol', 5, 0.5, 30],
+    ['gpt-5.6-terra', 2.5, 0.25, 15],
+    ['gpt-5.6-luna', 1, 0.1, 6],
+  ])('prices %s cached input at the published 90%% discount', (model, inputRate, cachedRate, outputRate) => {
+    const usage = {
+      prompt_tokens: 2_000,
+      completion_tokens: 1_000,
+      prompt_tokens_details: { cached_tokens: 500, cache_write_tokens: 0 },
+    };
+
+    expect(calculateOpenAIUsageCost(model, {}, usage)).toBeCloseTo(
+      (1_500 * inputRate + 500 * cachedRate + 1_000 * outputRate) / 1e6,
+      10,
+    );
+  });
+
+  it.each([
+    ['gpt-5.6', 5, 30],
+    ['gpt-5.6-sol', 5, 30],
+    ['gpt-5.6-terra', 2.5, 15],
+    ['gpt-5.6-luna', 1, 6],
+  ])('prices %s image input tokens at the text input rate', (model, inputRate, outputRate) => {
+    expect(
+      calculateOpenAIUsageCost(
+        model,
+        {},
+        {
+          input_tokens: 1_000,
+          output_tokens: 100,
+          input_tokens_details: { image_tokens: 200, cache_write_tokens: 0 },
+        },
+      ),
+    ).toBeCloseTo((1_000 * inputRate + 100 * outputRate) / 1e6, 10);
+  });
+
+  it('prices cached GPT-5.6 image input tokens at the cached text input rate', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6',
+        {},
+        {
+          input_tokens: 1_000,
+          output_tokens: 100,
+          input_tokens_details: {
+            text_tokens: 800,
+            image_tokens: 200,
+            cached_tokens: 200,
+            cached_tokens_details: { image_tokens: 200 },
+            cache_write_tokens: 0,
+          },
+        },
+      ),
+    ).toBeCloseTo((800 * 5 + 200 * 0.5 + 100 * 30) / 1e6, 10);
+  });
+
+  it('prices GPT-5.6 explicit cache writes at 1.25x input', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6',
+        {},
+        {
+          input_tokens: 2_000,
+          output_tokens: 1_000,
+          input_tokens_details: { cached_tokens: 500, cache_write_tokens: 250 },
+        },
+      ),
+    ).toBeCloseTo((1_250 * 5 + 500 * 0.5 + 250 * 6.25 + 1_000 * 30) / 1e6, 10);
+  });
+
+  it.each([
+    'gpt-5.6',
+    'gpt-5.6-sol',
+    'gpt-5.6-terra',
+    'gpt-5.6-luna',
+  ])('omits %s cost when raw usage lacks cache-write tokens', (model) => {
+    expect(
+      calculateOpenAIUsageCost(
+        model,
+        {},
+        {
+          input_tokens: 2_000,
+          output_tokens: 1_000,
+          input_tokens_details: { cached_tokens: 500 },
+        },
+      ),
+    ).toBeUndefined();
+  });
+
+  it('uses a custom GPT-5.6 input cost when cache-write tokens are unavailable', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6',
+        { inputCost: 2 / 1e6 },
+        {
+          input_tokens: 2_000,
+          output_tokens: 1_000,
+          input_tokens_details: { cached_tokens: 500 },
+        },
+      ),
+    ).toBeCloseTo((2_000 * 2 + 1_000 * 30) / 1e6, 10);
+  });
+
+  it('accepts an explicit top-level zero cache-write count for GPT-5.6', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6-terra',
+        {},
+        {
+          input_tokens: 2_000,
+          output_tokens: 1_000,
+          cache_write_input_tokens: 0,
+        },
+      ),
+    ).toBeCloseTo((2_000 * 2.5 + 1_000 * 15) / 1e6, 10);
+  });
+
+  it.each([
+    [{ apiHost: 'eu.api.openai.com' }, {}],
+    [{ apiBaseUrl: 'https://us.api.openai.com/v1' }, {}],
+    [{}, { apiUrl: 'https://eu.api.openai.com/v1' }],
+  ])('applies the GPT-5.6 regional processing uplift', (config, options) => {
+    const usage = {
+      input_tokens: 2_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 500, cache_write_tokens: 250 },
+    };
+
+    expect(calculateOpenAIUsageCost('gpt-5.6', config, usage, options)).toBeCloseTo(
+      ((1_250 * 5 + 500 * 0.5 + 250 * 6.25 + 1_000 * 30) / 1e6) * 1.1,
+      10,
+    );
+  });
+
+  it.each([
+    'gpt-5.4',
+    'gpt-5.4-2026-03-05',
+    'gpt-5.4-mini',
+    'gpt-5.4-mini-2026-03-17',
+    'gpt-5.4-nano',
+    'gpt-5.4-nano-2026-03-17',
+    'gpt-5.4-pro',
+    'gpt-5.4-pro-2026-03-05',
+    'gpt-5.5',
+    'gpt-5.5-2026-04-23',
+    'gpt-5.5-pro',
+    'gpt-5.5-pro-2026-04-23',
+    'gpt-5.6',
+    'gpt-5.6-sol',
+    'gpt-5.6-terra',
+    'gpt-5.6-luna',
+  ])('applies the regional processing uplift to eligible model %s', (model) => {
+    const usage = {
+      input_tokens: 2_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 500, cache_write_tokens: 0 },
+    };
+    const standardCost = calculateOpenAIUsageCost(model, {}, usage);
+
+    expect(standardCost).toBeDefined();
+    expect(calculateOpenAIUsageCost(model, { apiHost: 'eu.api.openai.com' }, usage)).toBeCloseTo(
+      standardCost! * 1.1,
+      10,
+    );
+  });
+
+  it('does not apply the regional processing uplift to models released before the cutoff', () => {
+    const usage = { input_tokens: 2_000, output_tokens: 1_000 };
+    const standardCost = calculateOpenAIUsageCost('gpt-5.2', {}, usage);
+
+    expect(standardCost).toBeDefined();
+    expect(
+      calculateOpenAIUsageCost('gpt-5.2', { apiHost: 'eu.api.openai.com' }, usage),
+    ).toBeCloseTo(standardCost!, 10);
+  });
+
+  it('preserves GPT-5.6 custom costs while uplifting remaining regional rates', () => {
+    const usage = {
+      input_tokens: 2_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 500, cache_write_tokens: 250 },
+    };
+
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6',
+        {
+          apiHost: 'eu.api.openai.com',
+          inputCost: 2 / 1e6,
+        },
+        usage,
+      ),
+    ).toBeCloseTo((2_000 * 2 + 1_000 * 30 * 1.1) / 1e6, 10);
+  });
+
+  it.each([
+    'proxy.api.openai.com',
+    'au.api.openai.com',
+  ])('does not apply the GPT-5.6 regional uplift to %s', (apiHost) => {
+    const usage = {
+      input_tokens: 2_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 500, cache_write_tokens: 250 },
+    };
+
+    expect(calculateOpenAIUsageCost('gpt-5.6', { apiHost }, usage)).toBeCloseTo(
+      (1_250 * 5 + 500 * 0.5 + 250 * 6.25 + 1_000 * 30) / 1e6,
+      10,
+    );
+  });
+
+  it('omits GPT-5.6 cost when summarized usage lacks cache-write tokens', () => {
+    expect(
+      calculateOpenAIUsageCostFromTokenUsage('gpt-5.6-sol', {
+        prompt: 2_000,
+        completion: 1_000,
+        cached: 500,
+      }),
+    ).toBeUndefined();
+  });
+
+  it('prices GPT-5.6 summarized usage when cache-write tokens are known', () => {
+    expect(
+      calculateOpenAIUsageCostFromTokenUsage('gpt-5.6-sol', {
+        prompt: 2_000,
+        completion: 1_000,
+        cached: 500,
+        completionDetails: { cacheCreationInputTokens: 250 },
+      }),
+    ).toBeCloseTo((1_250 * 5 + 500 * 0.5 + 250 * 6.25 + 1_000 * 30) / 1e6, 10);
+  });
+
+  it('uses GPT-5.6 Flex long-context rates and rejects unsupported Priority long context', () => {
+    const usage = {
+      input_tokens: 300_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 100_000, cache_write_tokens: 50_000 },
+    };
+
+    expect(calculateOpenAIUsageCost('gpt-5.6-terra', {}, usage)).toBeCloseTo(
+      (150_000 * 5 + 100_000 * 0.5 + 50_000 * 6.25 + 1_000 * 22.5) / 1e6,
+      10,
+    );
+    expect(
+      calculateOpenAIUsageCost('gpt-5.6-terra', {}, usage, { serviceTier: 'flex' }),
+    ).toBeCloseTo((150_000 * 2.5 + 100_000 * 0.25 + 50_000 * 3.125 + 1_000 * 11.25) / 1e6, 10);
+    expect(
+      calculateOpenAIUsageCost('gpt-5.6-terra', {}, usage, { serviceTier: 'priority' }),
+    ).toBeUndefined();
+  });
+
+  it('uses GPT-5.6 Priority rates through the 272K input limit', () => {
+    const usage = {
+      input_tokens: 272_000,
+      output_tokens: 1_000,
+      input_tokens_details: { cached_tokens: 100_000, cache_write_tokens: 50_000 },
+    };
+
+    expect(
+      calculateOpenAIUsageCost('gpt-5.6-terra', {}, usage, { serviceTier: 'priority' }),
+    ).toBeCloseTo((122_000 * 5 + 100_000 * 0.5 + 50_000 * 6.25 + 1_000 * 30) / 1e6, 10);
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.6-terra',
+        {},
+        { ...usage, input_tokens: 272_001 },
+        { serviceTier: 'priority' },
+      ),
+    ).toBeUndefined();
+  });
+
+  it('prices chat-latest cached input at the published discount', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'chat-latest',
+        {},
+        {
+          prompt_tokens: 2_000,
+          completion_tokens: 1_000,
+          prompt_tokens_details: { cached_tokens: 500 },
+        },
+      ),
+    ).toBeCloseTo((1_500 * 5 + 500 * 0.5 + 1_000 * 30) / 1e6, 10);
+  });
+
+  it('uses returned service tiers when pricing flex and priority work', () => {
+    const usage = {
+      input_tokens: 1_000,
+      output_tokens: 100,
+      input_tokens_details: { cached_tokens: 400 },
+    };
+
+    expect(calculateOpenAIUsageCost('gpt-5-mini', {}, usage, { serviceTier: 'flex' })).toBeCloseTo(
+      (600 * 0.125 + 400 * 0.0125 + 100 * 1) / 1e6,
+      10,
+    );
+    expect(
+      calculateOpenAIUsageCost('gpt-5-mini', {}, usage, { serviceTier: 'priority' }),
+    ).toBeCloseTo((600 * 0.45 + 400 * 0.045 + 100 * 3.6) / 1e6, 10);
+  });
+
+  it('uses current long-context flex rates for supported pro models', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5.5-pro',
+        {},
+        {
+          input_tokens: 300_000,
+          output_tokens: 1_000,
+        },
+        { serviceTier: 'flex' },
+      ),
+    ).toBeCloseTo((300_000 * 30 + 1_000 * 135) / 1e6, 10);
+  });
+
+  it('does not invent flex pricing for unsupported models', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-4.1',
+        {},
+        {
+          input_tokens: 1_000,
+          output_tokens: 100,
+        },
+        { serviceTier: 'flex' },
+      ),
+    ).toBeUndefined();
+  });
+
+  it('applies the Batch API discount to standard rates', () => {
+    const usage = {
+      input_tokens: 1_000,
+      output_tokens: 100,
+      input_tokens_details: { cached_tokens: 400 },
+    };
+
+    expect(calculateOpenAIUsageCost('gpt-5-mini', {}, usage, { serviceTier: 'batch' })).toBeCloseTo(
+      (600 * 0.125 + 400 * 0.0125 + 100 * 1) / 1e6,
+      10,
+    );
+    expect(
+      calculateOpenAIUsageCost(
+        'text-embedding-3-large',
+        {},
+        { prompt_tokens: 10, total_tokens: 10 },
+        { serviceTier: 'batch' },
+      ),
+    ).toBeCloseTo((10 * 0.065) / 1e6, 12);
+  });
+
+  it('keeps cached responses for unknown models unpriced', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'third-party/model',
+        {},
+        {
+          prompt_tokens: 10,
+          completion_tokens: 5,
+        },
+        { cachedResponse: true },
+      ),
+    ).toBeUndefined();
+  });
+
+  it('prices audio text and audio tokens separately', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-4o-mini-audio-preview',
+      {},
+      {
+        prompt_tokens: 30,
+        completion_tokens: 23,
+        prompt_tokens_details: {
+          text_tokens: 21,
+          audio_tokens: 9,
+        },
+        completion_tokens_details: {
+          text_tokens: 16,
+          audio_tokens: 7,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo((21 * 0.15 + 9 * 10 + 16 * 0.6 + 7 * 20) / 1e6, 10);
+  });
+
+  it('uses current gpt-realtime-mini multimodal and cached rates', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-realtime-mini',
+      {},
+      {
+        input_tokens: 1_030,
+        output_tokens: 30,
+        input_tokens_details: {
+          text_tokens: 1_000,
+          audio_tokens: 20,
+          image_tokens: 10,
+          cached_tokens: 100,
+        },
+        output_tokens_details: {
+          text_tokens: 20,
+          audio_tokens: 10,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo(
+      (900 * 0.6 + 100 * 0.06 + 20 * 10 + 10 * 0.8 + 20 * 2.4 + 10 * 20) / 1e6,
+      10,
+    );
+  });
+
+  it('uses current gpt-realtime-2 multimodal and cached rates', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-realtime-2',
+      {},
+      {
+        input_tokens: 1_030,
+        output_tokens: 30,
+        input_tokens_details: {
+          text_tokens: 1_000,
+          audio_tokens: 20,
+          image_tokens: 10,
+          cached_tokens: 100,
+        },
+        output_tokens_details: {
+          text_tokens: 20,
+          audio_tokens: 10,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo(
+      (900 * 4 + 100 * 0.4 + 20 * 32 + 10 * 5 + 20 * 24 + 10 * 64) / 1e6,
+      10,
+    );
+  });
+
+  it('uses explicit cached modality splits when realtime payloads provide them', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-realtime-mini',
+      {},
+      {
+        input_tokens: 1_030,
+        output_tokens: 30,
+        input_token_details: {
+          text_tokens: 1_000,
+          audio_tokens: 20,
+          image_tokens: 10,
+          cached_tokens: 100,
+          cached_tokens_details: {
+            text_tokens: 70,
+            audio_tokens: 20,
+            image_tokens: 10,
+          },
+        },
+        output_token_details: {
+          text_tokens: 20,
+          audio_tokens: 10,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo(
+      (930 * 0.6 + 70 * 0.06 + 20 * 0.3 + 10 * 0.08 + 20 * 2.4 + 10 * 20) / 1e6,
+      10,
+    );
+  });
+
+  it('uses realtime singular usage detail keys when pricing', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-realtime-mini',
+      {},
+      {
+        input_tokens: 30,
+        output_tokens: 23,
+        input_token_details: {
+          text_tokens: 21,
+          audio_tokens: 9,
+        },
+        output_token_details: {
+          text_tokens: 16,
+          audio_tokens: 7,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo((21 * 0.6 + 9 * 10 + 16 * 2.4 + 7 * 20) / 1e6, 10);
+  });
+
+  it('prices GPT Image usage from the returned token ledger', () => {
+    const cost = calculateOpenAIUsageCost(
+      'gpt-image-1.5',
+      {},
+      {
+        input_tokens: 222,
+        output_tokens: 453,
+        input_tokens_details: {
+          text_tokens: 28,
+          image_tokens: 194,
+        },
+        output_tokens_details: {
+          text_tokens: 181,
+          image_tokens: 272,
+        },
+      },
+      {},
+    );
+
+    expect(cost).toBeCloseTo((28 * 5 + 194 * 8 + 181 * 10 + 272 * 32) / 1e6, 10);
+  });
+
+  it('does not claim exact GPT Image 1.5 cost without an output-token breakdown', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-image-1.5',
+        {},
+        {
+          input_tokens: 222,
+          output_tokens: 453,
+          input_tokens_details: {
+            text_tokens: 28,
+            image_tokens: 194,
+          },
+        },
+        {},
+      ),
+    ).toBeUndefined();
+  });
+
+  it('prices embeddings from prompt tokens', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'text-embedding-3-large',
+        {},
+        {
+          prompt_tokens: 10,
+          total_tokens: 10,
+        },
+        {},
+      ),
+    ).toBeCloseTo((10 * 0.13) / 1e6, 12);
+  });
+
+  it('does not bill promptfoo cache hits again', () => {
+    expect(
+      calculateOpenAIUsageCost(
+        'gpt-5-mini',
+        {},
+        {
+          prompt_tokens: 1_000,
+          completion_tokens: 100,
+        },
+        { cachedResponse: true },
+      ),
+    ).toBe(0);
+  });
+
+  it('adds observable call-based tool fees', () => {
+    expect(
+      calculateObservableOpenAIToolCost(
+        {
+          output: [
+            { type: 'web_search_call', action: { type: 'search' } },
+            { type: 'web_search_call', action: { type: 'search' } },
+            { type: 'file_search_call' },
+          ],
+        },
+        'gpt-5-mini',
+        { tools: [{ type: 'web_search_preview' }] },
+      ),
+    ).toBeCloseTo(0.0225, 10);
+    expect(
+      calculateObservableOpenAIToolCost(
+        {
+          output: [{ type: 'web_search_call', action: { type: 'search' } }],
+        },
+        'gpt-4o',
+        { tools: [{ type: 'web_search_preview' }] },
+      ),
+    ).toBeCloseTo(0.025, 10);
+  });
+
+  it('uses non-preview web search pricing when configured', () => {
+    expect(
+      calculateObservableOpenAIToolCost(
+        {
+          output: [{ type: 'web_search_call', action: { type: 'search' } }],
+        },
+        'gpt-4o',
+        { tools: [{ type: 'web_search' }] },
+      ),
+    ).toBeCloseTo(0.01, 10);
+  });
+
+  it('does not charge non-search web actions', () => {
+    expect(
+      calculateObservableOpenAIToolCost(
+        {
+          output: [
+            { type: 'web_search_call', action: { type: 'open_page' } },
+            { type: 'web_search_call', action: { type: 'find_in_page' } },
+          ],
+        },
+        'o3',
+        { tools: [{ type: 'web_search_preview' }] },
+      ),
+    ).toBe(0);
+  });
+});

test/providers/openai/chatkit-pool.test.ts

@@ -0,0 +1,443 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { ChatKitBrowserPool } from '../../../src/providers/openai/chatkit-pool';
+
+// Create hoisted mocks to access them in tests
+const mockPage = vi.hoisted(() => ({
+  goto: vi.fn().mockResolvedValue(undefined),
+  waitForFunction: vi.fn().mockResolvedValue(undefined),
+  reload: vi.fn().mockResolvedValue(undefined),
+}));
+
+const mockContext = vi.hoisted(() => ({
+  newPage: vi.fn().mockResolvedValue(mockPage),
+  close: vi.fn().mockResolvedValue(undefined),
+  setDefaultTimeout: vi.fn(),
+}));
+
+const mockBrowser = vi.hoisted(() => ({
+  newContext: vi.fn().mockResolvedValue(mockContext),
+  close: vi.fn().mockResolvedValue(undefined),
+}));
+
+const mockChromium = vi.hoisted(() => ({
+  launch: vi.fn().mockResolvedValue(mockBrowser),
+}));
+
+// Mock playwright
+vi.mock('playwright', () => ({
+  chromium: mockChromium,
+}));
+
+// Mock http server
+vi.mock('http', () => ({
+  createServer: vi.fn().mockReturnValue({
+    listen: vi.fn((_port: number, callback: () => void) => callback()),
+    address: vi.fn().mockReturnValue({ port: 3000 }),
+    close: vi.fn(),
+    once: vi.fn(), // Error handler registration
+  }),
+}));
+
+// Test constants
+const TEST_TEMPLATE_KEY = 'wf_test123:default:default';
+const TEST_HTML = '<html>test</html>';
+
+function resetPlaywrightMocks() {
+  mockPage.goto.mockReset().mockResolvedValue(undefined);
+  mockPage.waitForFunction.mockReset().mockResolvedValue(undefined);
+  mockPage.reload.mockReset().mockResolvedValue(undefined);
+  mockContext.newPage.mockReset().mockResolvedValue(mockPage);
+  mockContext.close.mockReset().mockResolvedValue(undefined);
+  mockContext.setDefaultTimeout.mockReset();
+  mockBrowser.newContext.mockReset().mockResolvedValue(mockContext);
+  mockBrowser.close.mockReset().mockResolvedValue(undefined);
+  mockChromium.launch.mockReset().mockResolvedValue(mockBrowser);
+}
+
+describe('ChatKitBrowserPool', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    resetPlaywrightMocks();
+    // Reset the singleton between tests
+    ChatKitBrowserPool.resetInstance();
+  });
+
+  afterEach(async () => {
+    // Use clearAllMocks instead of resetAllMocks to preserve mock implementations
+    vi.clearAllMocks();
+    // Ensure clean state after each test
+    ChatKitBrowserPool.resetInstance();
+  });
+
+  describe('getInstance', () => {
+    it('should return singleton instance', () => {
+      const instance1 = ChatKitBrowserPool.getInstance();
+      const instance2 = ChatKitBrowserPool.getInstance();
+
+      expect(instance1).toBe(instance2);
+    });
+
+    it('should use default config values', () => {
+      const instance = ChatKitBrowserPool.getInstance();
+
+      // Config should use defaults
+      expect((instance as any).config.maxConcurrency).toBe(4);
+      expect((instance as any).config.headless).toBe(true);
+    });
+
+    it('should respect custom config values', () => {
+      const instance = ChatKitBrowserPool.getInstance({
+        maxConcurrency: 8,
+        headless: false,
+      });
+
+      expect((instance as any).config.maxConcurrency).toBe(8);
+      expect((instance as any).config.headless).toBe(false);
+    });
+
+    it('should ignore config on subsequent calls (singleton already exists)', () => {
+      const _instance1 = ChatKitBrowserPool.getInstance({ maxConcurrency: 4 });
+      const instance2 = ChatKitBrowserPool.getInstance({ maxConcurrency: 10 });
+
+      // Should still be 4, not 10
+      expect((instance2 as any).config.maxConcurrency).toBe(4);
+    });
+  });
+
+  describe('resetInstance', () => {
+    it('should allow creating new instance after reset', () => {
+      const instance1 = ChatKitBrowserPool.getInstance({ maxConcurrency: 4 });
+      ChatKitBrowserPool.resetInstance();
+      const instance2 = ChatKitBrowserPool.getInstance({ maxConcurrency: 10 });
+
+      expect(instance1).not.toBe(instance2);
+      expect((instance2 as any).config.maxConcurrency).toBe(10);
+    });
+  });
+
+  describe('generateTemplateKey', () => {
+    it('should generate key from workflowId only', () => {
+      const key = ChatKitBrowserPool.generateTemplateKey('wf_abc123');
+      expect(key).toBe('wf_abc123:default:default');
+    });
+
+    it('should include version when provided', () => {
+      const key = ChatKitBrowserPool.generateTemplateKey('wf_abc123', '2');
+      expect(key).toBe('wf_abc123:2:default');
+    });
+
+    it('should include userId when provided', () => {
+      const key = ChatKitBrowserPool.generateTemplateKey('wf_abc123', undefined, 'user@test.com');
+      expect(key).toBe('wf_abc123:default:user@test.com');
+    });
+
+    it('should include all components', () => {
+      const key = ChatKitBrowserPool.generateTemplateKey('wf_abc123', '3', 'user@test.com');
+      expect(key).toBe('wf_abc123:3:user@test.com');
+    });
+  });
+
+  describe('setTemplate', () => {
+    it('should store the HTML template for a key', () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      const html = '<html><body>Test</body></html>';
+
+      instance.setTemplate(TEST_TEMPLATE_KEY, html);
+
+      expect((instance as any).templates.get(TEST_TEMPLATE_KEY)).toBe(html);
+    });
+
+    it('should store multiple templates for different keys', () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      const key1 = 'wf_workflow1:default:default';
+      const key2 = 'wf_workflow2:default:default';
+
+      instance.setTemplate(key1, '<html>workflow1</html>');
+      instance.setTemplate(key2, '<html>workflow2</html>');
+
+      expect((instance as any).templates.size).toBe(2);
+      expect((instance as any).templates.get(key1)).toBe('<html>workflow1</html>');
+      expect((instance as any).templates.get(key2)).toBe('<html>workflow2</html>');
+    });
+  });
+
+  describe('initialize', () => {
+    it('should start HTTP server and launch browser', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+
+      await instance.initialize();
+
+      expect((instance as any).initialized).toBe(true);
+      expect((instance as any).browser).toBe(mockBrowser);
+      expect((instance as any).server).not.toBeNull();
+    });
+
+    it('should not reinitialize if already initialized', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+
+      await instance.initialize();
+      await instance.initialize();
+
+      // Should only launch browser once
+      expect(mockChromium.launch).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle concurrent initialization calls', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+
+      // Multiple concurrent initialize calls
+      await Promise.all([instance.initialize(), instance.initialize(), instance.initialize()]);
+
+      // Should still only launch browser once
+      expect(mockChromium.launch).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('acquirePage', () => {
+    it('should create a new page when pool is empty', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      const pooledPage = await instance.acquirePage(TEST_TEMPLATE_KEY);
+
+      expect(pooledPage).toBeDefined();
+      expect(pooledPage.page).toBe(mockPage);
+      expect(pooledPage.context).toBe(mockContext);
+      expect(pooledPage.inUse).toBe(true);
+      expect(pooledPage.ready).toBe(true);
+      expect(pooledPage.templateKey).toBe(TEST_TEMPLATE_KEY);
+    });
+
+    it('should reuse existing page when available for same template', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 2 });
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      const page1 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      await instance.releasePage(page1);
+
+      const page2 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+
+      // Should reuse the same page
+      expect(page2).toBe(page1);
+    });
+
+    it('should create separate pages for different templates', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 4 });
+      const key1 = 'wf_workflow1:default:default';
+      const key2 = 'wf_workflow2:default:default';
+
+      instance.setTemplate(key1, '<html>workflow1</html>');
+      instance.setTemplate(key2, '<html>workflow2</html>');
+
+      const page1 = await instance.acquirePage(key1);
+      const page2 = await instance.acquirePage(key2);
+
+      expect(page1.templateKey).toBe(key1);
+      expect(page2.templateKey).toBe(key2);
+      expect(page1).not.toBe(page2);
+    });
+
+    it('should create multiple pages up to maxConcurrency', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 3 });
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      const pages = await Promise.all([
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+      ]);
+
+      expect(pages.length).toBe(3);
+      expect(instance.getStats().total).toBe(3);
+      expect(instance.getStats().inUse).toBe(3);
+    });
+
+    it('should throw error if template not registered', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+
+      await expect(instance.acquirePage('unregistered_key')).rejects.toThrow(
+        'Template not registered',
+      );
+    });
+  });
+
+  describe('releasePage', () => {
+    it('should mark page as not in use', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      const pooledPage = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      expect(pooledPage.inUse).toBe(true);
+
+      await instance.releasePage(pooledPage);
+      expect(pooledPage.inUse).toBe(false);
+    });
+
+    it('should reload page for fresh state', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      const pooledPage = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      await instance.releasePage(pooledPage);
+
+      expect(mockPage.reload).toHaveBeenCalledWith({ waitUntil: 'domcontentloaded' });
+    });
+
+    it('should handle multiple acquire/release cycles', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 1 });
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      // First acquire
+      const page1 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      expect(page1.inUse).toBe(true);
+      expect(instance.getStats().inUse).toBe(1);
+
+      // Release
+      await instance.releasePage(page1);
+      expect(page1.inUse).toBe(false);
+
+      // Acquire again - should get the same page
+      const page2 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      expect(page2).toBe(page1);
+      expect(page2.inUse).toBe(true);
+    });
+  });
+
+  describe('getStats', () => {
+    it('should return accurate pool statistics', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 3 });
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      expect(instance.getStats()).toEqual({ total: 0, inUse: 0, waiting: 0, templates: 1 });
+
+      const page1 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      expect(instance.getStats()).toEqual({ total: 1, inUse: 1, waiting: 0, templates: 1 });
+
+      const _page2 = await instance.acquirePage(TEST_TEMPLATE_KEY);
+      expect(instance.getStats()).toEqual({ total: 2, inUse: 2, waiting: 0, templates: 1 });
+
+      await instance.releasePage(page1);
+      expect(instance.getStats()).toEqual({ total: 2, inUse: 1, waiting: 0, templates: 1 });
+    });
+
+    it('should count multiple templates', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 4 });
+      instance.setTemplate('key1', '<html>1</html>');
+      instance.setTemplate('key2', '<html>2</html>');
+
+      expect(instance.getStats().templates).toBe(2);
+    });
+  });
+
+  describe('shutdown', () => {
+    it('should close all contexts and browser', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      await instance.acquirePage(TEST_TEMPLATE_KEY);
+      await instance.shutdown();
+
+      expect(mockContext.close).toHaveBeenCalled();
+      expect(mockBrowser.close).toHaveBeenCalled();
+      expect((instance as any).initialized).toBe(false);
+      expect((instance as any).browser).toBeNull();
+    });
+
+    it('should clear all pages from pool', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 3 });
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      await Promise.all([
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+        instance.acquirePage(TEST_TEMPLATE_KEY),
+      ]);
+
+      expect(instance.getStats().total).toBe(3);
+
+      await instance.shutdown();
+
+      expect(instance.getStats().total).toBe(0);
+    });
+
+    it('should clear templates', async () => {
+      const instance = ChatKitBrowserPool.getInstance();
+      instance.setTemplate(TEST_TEMPLATE_KEY, TEST_HTML);
+
+      expect(instance.getStats().templates).toBe(1);
+
+      await instance.shutdown();
+
+      expect(instance.getStats().templates).toBe(0);
+    });
+  });
+
+  describe('error handling', () => {
+    it('should throw helpful error when Playwright not installed', async () => {
+      mockChromium.launch.mockRejectedValueOnce(
+        new Error("Executable doesn't exist at /path/to/browser"),
+      );
+
+      ChatKitBrowserPool.resetInstance();
+
+      const newInstance = ChatKitBrowserPool.getInstance();
+
+      await expect(newInstance.initialize()).rejects.toThrow('Playwright browser not installed');
+    });
+  });
+
+  describe('template isolation', () => {
+    it('should not reuse pages across different templates', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 4 });
+      const key1 = 'wf_workflow1:default:default';
+      const key2 = 'wf_workflow2:default:default';
+
+      instance.setTemplate(key1, '<html>workflow1</html>');
+      instance.setTemplate(key2, '<html>workflow2</html>');
+
+      // Get a page for workflow1
+      const page1 = await instance.acquirePage(key1);
+      await instance.releasePage(page1);
+
+      // Request a page for workflow2 - should NOT get the workflow1 page
+      const page2 = await instance.acquirePage(key2);
+      expect(page2.templateKey).toBe(key2);
+      expect(page2).not.toBe(page1);
+    });
+
+    it('should only give released pages to waiters with matching template', async () => {
+      const instance = ChatKitBrowserPool.getInstance({ maxConcurrency: 2 });
+      const key1 = 'wf_workflow1:default:default';
+      const key2 = 'wf_workflow2:default:default';
+
+      instance.setTemplate(key1, '<html>workflow1</html>');
+      instance.setTemplate(key2, '<html>workflow2</html>');
+
+      // Get pages for both workflows
+      const page1 = await instance.acquirePage(key1);
+      const page2 = await instance.acquirePage(key2);
+
+      // Release workflow1 page
+      await instance.releasePage(page1);
+
+      // Start waiting for workflow2 (a second workflow2 page)
+      // Since maxConcurrency is 2 and both slots are taken (workflow1 released, workflow2 in use)
+      // The waiter should eventually get the workflow1 page when released? No!
+      // Actually with maxConcurrency: 2 and both pages created, we're at limit
+      // Let's just verify the released page1 doesn't get given to a workflow2 request
+
+      // Release workflow2 page
+      await instance.releasePage(page2);
+
+      // Now request workflow2 - should get page2 back, not page1
+      const page2Again = await instance.acquirePage(key2);
+      expect(page2Again).toBe(page2);
+      expect(page2Again.templateKey).toBe(key2);
+
+      // Request workflow1 - should get page1 back
+      const page1Again = await instance.acquirePage(key1);
+      expect(page1Again).toBe(page1);
+      expect(page1Again.templateKey).toBe(key1);
+    });
+  });
+});

test/providers/openai/chatkit.test.ts

@@ -0,0 +1,507 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { disableCache, enableCache } from '../../../src/cache';
+import {
+  cleanAssistantResponse,
+  OpenAiChatKitProvider,
+} from '../../../src/providers/openai/chatkit';
+import { mockProcessEnv } from '../../util/utils';
+
+// Mock Playwright - we don't want to actually launch browsers in unit tests
+vi.mock('playwright', () => ({
+  chromium: {
+    launch: vi.fn().mockResolvedValue({
+      newContext: vi.fn().mockResolvedValue({
+        newPage: vi.fn().mockResolvedValue({
+          goto: vi.fn().mockResolvedValue(undefined),
+          waitForFunction: vi.fn().mockResolvedValue(undefined),
+          waitForTimeout: vi.fn().mockResolvedValue(undefined),
+          evaluate: vi.fn().mockResolvedValue(undefined),
+          reload: vi.fn().mockResolvedValue(undefined),
+          frames: vi.fn().mockReturnValue([]),
+          on: vi.fn(), // Console event listener
+        }),
+        close: vi.fn().mockResolvedValue(undefined),
+      }),
+      close: vi.fn().mockResolvedValue(undefined),
+    }),
+  },
+}));
+
+// Mock http server
+vi.mock('http', () => ({
+  createServer: vi.fn().mockReturnValue({
+    listen: vi.fn((_port: number, callback: () => void) => callback()),
+    address: vi.fn().mockReturnValue({ port: 3000 }),
+    close: vi.fn(),
+    once: vi.fn(), // For error event handler
+  }),
+}));
+
+// Helper to access the generated HTML (we test via the provider's internal HTML generation)
+function getGeneratedHTML(provider: OpenAiChatKitProvider): string {
+  // Access private method via casting
+  const config = (provider as any).chatKitConfig;
+  const apiKey = 'test-key';
+  const workflowId = config.workflowId;
+  const version = config.version;
+  const userId = config.userId;
+
+  // Generate HTML the same way the provider does internally
+  const versionClause = version ? `, version: '${version}'` : '';
+  return `<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>ChatKit Eval</title>
+</head>
+<body>
+  <openai-chatkit id="chatkit"></openai-chatkit>
+
+  <script src="https://cdn.platform.openai.com/deployments/chatkit/chatkit.js"></script>
+
+  <script>
+    window.__state = { ready: false, responses: [], threadId: null, error: null, responding: false };
+
+    async function init() {
+      const chatkit = document.getElementById('chatkit');
+
+      // Wait for element to be ready
+      let attempts = 0;
+      while (typeof chatkit.setOptions !== 'function' && attempts < 100) {
+        await new Promise(r => setTimeout(r, 100));
+        attempts++;
+      }
+
+      if (typeof chatkit.setOptions !== 'function') {
+        window.__state.error = 'ChatKit component failed to initialize';
+        return;
+      }
+
+      let cachedSecret = null;
+
+      chatkit.setOptions({
+        api: {
+          getClientSecret: async (existing) => {
+            if (existing) return existing;
+            if (cachedSecret) return cachedSecret;
+
+            const res = await fetch('https://api.openai.com/v1/chatkit/sessions', {
+              method: 'POST',
+              headers: {
+                'Authorization': 'Bearer ${apiKey}',
+                'Content-Type': 'application/json',
+                'OpenAI-Beta': 'chatkit_beta=v1',
+                'X-OpenAI-Originator': 'promptfoo'
+              },
+              body: JSON.stringify({
+                workflow: { id: '${workflowId}'${versionClause} },
+                user: '${userId}'
+              })
+            });
+
+            if (!res.ok) {
+              const text = await res.text();
+              throw new Error('Session failed: ' + res.status + ' ' + text);
+            }
+
+            const data = await res.json();
+            cachedSecret = data.client_secret;
+            return cachedSecret;
+          }
+        },
+        header: { enabled: false },
+        history: { enabled: false },
+      });
+
+      chatkit.addEventListener('chatkit.ready', () => {
+        window.__state.ready = true;
+      });
+
+      chatkit.addEventListener('chatkit.error', (e) => {
+        window.__state.error = e.detail.error?.message || 'Unknown error';
+      });
+
+      chatkit.addEventListener('chatkit.thread.change', (e) => {
+        window.__state.threadId = e.detail.threadId;
+      });
+
+      chatkit.addEventListener('chatkit.response.start', () => {
+        window.__state.responding = true;
+      });
+
+      chatkit.addEventListener('chatkit.response.end', () => {
+        window.__state.responding = false;
+        window.__state.responses.push({ timestamp: Date.now() });
+      });
+
+      window.__chatkit = chatkit;
+    }
+
+    init().catch(e => {
+      window.__state.error = e.message;
+    });
+  </script>
+</body>
+</html>`;
+}
+
+describe('OpenAiChatKitProvider', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    disableCache();
+  });
+
+  afterEach(() => {
+    // Use clearAllMocks instead of resetAllMocks to preserve mock implementations
+    vi.clearAllMocks();
+    enableCache();
+  });
+
+  describe('constructor', () => {
+    it('should create provider with workflow ID', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(provider.id()).toBe('openai:chatkit:wf_test123');
+    });
+
+    it('should use workflowId from config if provided', () => {
+      const provider = new OpenAiChatKitProvider('default', {
+        config: {
+          apiKey: 'test-key',
+          workflowId: 'wf_custom',
+        },
+      });
+
+      expect(provider.id()).toBe('openai:chatkit:wf_custom');
+    });
+
+    it('should include version in id() when provided', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: {
+          apiKey: 'test-key',
+          version: '3',
+        },
+      });
+
+      expect(provider.id()).toBe('openai:chatkit:wf_test123:3');
+    });
+
+    it('should use default timeout of 120000ms', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      // Access private config via any
+      expect((provider as any).chatKitConfig.timeout).toBe(120000);
+    });
+
+    it('should use custom timeout when provided', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          timeout: 60000,
+        },
+      });
+
+      expect((provider as any).chatKitConfig.timeout).toBe(60000);
+    });
+
+    it('should default headless to true', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect((provider as any).chatKitConfig.headless).toBe(true);
+    });
+
+    it('should allow headless to be set to false', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          headless: false,
+        },
+      });
+
+      expect((provider as any).chatKitConfig.headless).toBe(false);
+    });
+  });
+
+  describe('toString', () => {
+    it('should return descriptive string', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(provider.toString()).toBe('[OpenAI ChatKit Provider wf_test123]');
+    });
+  });
+
+  describe('callApi', () => {
+    it('should return error when API key is missing', async () => {
+      const restoreEnv = mockProcessEnv({ OPENAI_API_KEY: undefined });
+      try {
+        const provider = new OpenAiChatKitProvider('wf_test', {});
+
+        const result = await provider.callApi('test prompt');
+
+        expect(result.error).toContain('API key');
+      } finally {
+        restoreEnv();
+      }
+    });
+  });
+
+  describe('cleanup', () => {
+    it('should close browser resources', async () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      // Initialize first (will use mocked playwright)
+      await (provider as any).initialize();
+
+      // Then cleanup
+      await provider.cleanup();
+
+      // Verify state is reset
+      expect((provider as any).browser).toBeNull();
+      expect((provider as any).context).toBeNull();
+      expect((provider as any).page).toBeNull();
+      expect((provider as any).server).toBeNull();
+      expect((provider as any).initialized).toBe(false);
+    });
+
+    it('should handle cleanup when not initialized', async () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      // Should not throw when cleaning up uninitialized provider
+      await expect(provider.cleanup()).resolves.toBeUndefined();
+    });
+  });
+
+  describe('HTML template generation', () => {
+    it('should include responding state tracking in window.__state', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const html = getGeneratedHTML(provider);
+
+      // Verify the responding state is included
+      expect(html).toContain('responding: false');
+      expect(html).toContain('window.__state.responding = true');
+      expect(html).toContain('window.__state.responding = false');
+    });
+
+    it('should have chatkit.response.start event listener', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const html = getGeneratedHTML(provider);
+
+      // Verify the response.start listener is included for multi-step workflow tracking
+      expect(html).toContain("chatkit.addEventListener('chatkit.response.start'");
+    });
+
+    it('should have chatkit.response.end event listener', () => {
+      const provider = new OpenAiChatKitProvider('wf_test123', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const html = getGeneratedHTML(provider);
+
+      // Verify the response.end listener is included
+      expect(html).toContain("chatkit.addEventListener('chatkit.response.end'");
+    });
+  });
+
+  describe('configuration validation', () => {
+    it('should reject invalid workflowId format', () => {
+      // The validateWorkflowId function in the source code rejects invalid formats
+      // This gets triggered during HTML generation, not during construction
+      const provider = new OpenAiChatKitProvider('invalid-workflow', {
+        config: { apiKey: 'test-key' },
+      });
+
+      // The validation happens when HTML is generated, which we can test via the helper
+      // Invalid workflow ID should not match the expected pattern
+      expect((provider as any).chatKitConfig.workflowId).toBe('invalid-workflow');
+    });
+
+    it('should accept valid workflowId format', () => {
+      const provider = new OpenAiChatKitProvider('wf_abc123XYZ', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect((provider as any).chatKitConfig.workflowId).toBe('wf_abc123XYZ');
+    });
+
+    it('should handle empty workflowId', () => {
+      const provider = new OpenAiChatKitProvider('', {
+        config: { apiKey: 'test-key', workflowId: 'wf_fromconfig' },
+      });
+
+      // Should use workflowId from config when empty string passed
+      expect((provider as any).chatKitConfig.workflowId).toBe('wf_fromconfig');
+    });
+
+    it('should handle special characters in userId', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          userId: 'user@example.com',
+        },
+      });
+
+      // userId with @ should be accepted
+      expect((provider as any).chatKitConfig.userId).toBe('user@example.com');
+    });
+
+    it('should handle version with dots and dashes', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          version: '1.2.3-beta',
+        },
+      });
+
+      expect((provider as any).chatKitConfig.version).toBe('1.2.3-beta');
+    });
+
+    it('should use consistent default userId across instances', () => {
+      const provider1 = new OpenAiChatKitProvider('wf_test1', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const provider2 = new OpenAiChatKitProvider('wf_test2', {
+        config: { apiKey: 'test-key' },
+      });
+
+      // Both should have the same default userId for template consistency
+      expect((provider1 as any).chatKitConfig.userId).toBe((provider2 as any).chatKitConfig.userId);
+    });
+  });
+
+  describe('cleanAssistantResponse', () => {
+    it('should return empty string for empty input', () => {
+      expect(cleanAssistantResponse('')).toBe('');
+    });
+
+    it('should return empty string for null/undefined input', () => {
+      expect(cleanAssistantResponse(null as any)).toBe('');
+      expect(cleanAssistantResponse(undefined as any)).toBe('');
+    });
+
+    it('should remove Cloudflare scripts', () => {
+      const input = 'Hello world(function(){console.log("cf")})();';
+      expect(cleanAssistantResponse(input)).toBe('Hello world');
+    });
+
+    it('should remove approval UI text from end of response', () => {
+      const input =
+        'Here is your response\nApproval required\nDoes this work for you?\nApprove\nReject';
+      expect(cleanAssistantResponse(input)).toBe('Here is your response');
+    });
+
+    it('should remove approval UI text with varying whitespace', () => {
+      const input =
+        'Here is your response\n\nApproval required\n\nDoes this work for you?\n\nApprove\n\nReject';
+      expect(cleanAssistantResponse(input)).toBe('Here is your response');
+    });
+
+    it('should return empty string when response starts with "You said:"', () => {
+      const input = 'You said: hello world';
+      expect(cleanAssistantResponse(input)).toBe('');
+    });
+
+    it('should remove "You said:" and everything after it', () => {
+      const input = 'Assistant response here\nYou said: some user input';
+      expect(cleanAssistantResponse(input)).toBe('Assistant response here');
+    });
+
+    it('should strip JSON prefix when followed by substantial text', () => {
+      const jsonPrefix = '{"classification": "return_item"}';
+      const substantialText =
+        'I can help you with your return request. Please provide your order number and reason for the return.';
+      const input = `${jsonPrefix} ${substantialText}`;
+      expect(cleanAssistantResponse(input)).toBe(substantialText);
+    });
+
+    it('should preserve JSON when it is the only response', () => {
+      const input = '{"classification": "return_item"}';
+      expect(cleanAssistantResponse(input)).toBe('{"classification": "return_item"}');
+    });
+
+    it('should preserve JSON when followed by short text (< 50 chars)', () => {
+      const input = '{"action": "query"} Short response';
+      expect(cleanAssistantResponse(input)).toBe('{"action": "query"} Short response');
+    });
+
+    it('should handle complex nested cleanup scenarios', () => {
+      const input =
+        '(function(){})();Here is your response\nApproval required\nDoes this work for you?\nApprove\nReject';
+      expect(cleanAssistantResponse(input)).toBe('Here is your response');
+    });
+
+    it('should trim whitespace from result', () => {
+      const input = '  Hello world  ';
+      expect(cleanAssistantResponse(input)).toBe('Hello world');
+    });
+  });
+
+  describe('approval handling configuration', () => {
+    it('should default approvalHandling to auto-approve', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect((provider as any).chatKitConfig.approvalHandling).toBe('auto-approve');
+    });
+
+    it('should allow approvalHandling to be set to auto-reject', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          approvalHandling: 'auto-reject',
+        },
+      });
+
+      expect((provider as any).chatKitConfig.approvalHandling).toBe('auto-reject');
+    });
+
+    it('should allow approvalHandling to be set to skip', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          approvalHandling: 'skip',
+        },
+      });
+
+      expect((provider as any).chatKitConfig.approvalHandling).toBe('skip');
+    });
+
+    it('should default maxApprovals to 5', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect((provider as any).chatKitConfig.maxApprovals).toBe(5);
+    });
+
+    it('should allow custom maxApprovals', () => {
+      const provider = new OpenAiChatKitProvider('wf_test', {
+        config: {
+          apiKey: 'test-key',
+          maxApprovals: 10,
+        },
+      });
+
+      expect((provider as any).chatKitConfig.maxApprovals).toBe(10);
+    });
+  });
+});

test/providers/openai/codexDefaults.test.ts

@@ -0,0 +1,523 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { providerRegistry } from '../../../src/providers/providerRegistry';
+import { createDeferred, mockProcessEnv } from '../../util/utils';
+
+import type { OpenAICodexSDKProvider } from '../../../src/providers/openai/codex-sdk';
+
+type CodexProviderBundle = {
+  gradingProvider: OpenAICodexSDKProvider;
+  gradingJsonProvider: OpenAICodexSDKProvider;
+  webSearchProvider: OpenAICodexSDKProvider;
+};
+
+// Each new bundle replaces the LRU's oldest entry once we add 32 keys, so the loop
+// below evicts whichever bundle was most recently used before the loop started.
+function fillCacheToOverflow(getProviders: (env: { CODEX_API_KEY: string }) => unknown): void {
+  for (let index = 1; index <= 32; index++) {
+    getProviders({ CODEX_API_KEY: `codex-key-${index}` });
+  }
+}
+
+function spyAndStubShutdowns(providers: CodexProviderBundle): {
+  grading: ReturnType<typeof vi.spyOn>;
+  gradingJson: ReturnType<typeof vi.spyOn>;
+  webSearch: ReturnType<typeof vi.spyOn>;
+} {
+  return {
+    grading: vi.spyOn(providers.gradingProvider, 'shutdown').mockResolvedValue(undefined),
+    gradingJson: vi.spyOn(providers.gradingJsonProvider, 'shutdown').mockResolvedValue(undefined),
+    webSearch: vi.spyOn(providers.webSearchProvider, 'shutdown').mockResolvedValue(undefined),
+  };
+}
+
+const mockGetDirectory = vi.hoisted(() => vi.fn(() => process.cwd()));
+const mockResolvePackageEntryPoint = vi.hoisted(() =>
+  vi.fn<(packageName: string, baseDir: string) => string | null>(() => '/tmp/codex-sdk/index.js'),
+);
+
+vi.mock('../../../src/esm', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('../../../src/esm')>()),
+  getDirectory: mockGetDirectory,
+  resolvePackageEntryPoint: mockResolvePackageEntryPoint,
+}));
+
+describe('Codex default providers', () => {
+  let codexHome: string;
+  let originalCodexApiKey: string | undefined;
+  let originalCodexHome: string | undefined;
+  let originalOpenAiApiKey: string | undefined;
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    mockGetDirectory.mockReset();
+    mockResolvePackageEntryPoint.mockReset();
+    mockGetDirectory.mockReturnValue(process.cwd());
+    mockResolvePackageEntryPoint.mockReturnValue('/tmp/codex-sdk/index.js');
+
+    originalCodexApiKey = process.env.CODEX_API_KEY;
+    originalCodexHome = process.env.CODEX_HOME;
+    originalOpenAiApiKey = process.env.OPENAI_API_KEY;
+    mockProcessEnv({ CODEX_API_KEY: undefined });
+    mockProcessEnv({ OPENAI_API_KEY: undefined });
+
+    codexHome = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-codex-defaults-'));
+    mockProcessEnv({ CODEX_HOME: codexHome });
+
+    const { clearCodexDefaultProvidersForTesting } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+    clearCodexDefaultProvidersForTesting();
+  });
+
+  afterEach(async () => {
+    const { clearCodexDefaultProvidersForTesting } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+    clearCodexDefaultProvidersForTesting();
+    await providerRegistry.shutdownAll();
+    vi.useRealTimers();
+    vi.resetAllMocks();
+
+    fs.rmSync(codexHome, { force: true, recursive: true });
+
+    if (originalCodexApiKey === undefined) {
+      mockProcessEnv({ CODEX_API_KEY: undefined });
+    } else {
+      mockProcessEnv({ CODEX_API_KEY: originalCodexApiKey });
+    }
+
+    if (originalCodexHome === undefined) {
+      mockProcessEnv({ CODEX_HOME: undefined });
+    } else {
+      mockProcessEnv({ CODEX_HOME: originalCodexHome });
+    }
+
+    if (originalOpenAiApiKey === undefined) {
+      mockProcessEnv({ OPENAI_API_KEY: undefined });
+    } else {
+      mockProcessEnv({ OPENAI_API_KEY: originalOpenAiApiKey });
+    }
+  });
+
+  it('returns true when CODEX_API_KEY exists and the Codex SDK package can be resolved', async () => {
+    mockProcessEnv({ CODEX_API_KEY: 'test-codex-key' });
+
+    const { hasCodexDefaultCredentials } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    expect(hasCodexDefaultCredentials()).toBe(true);
+  });
+
+  it('returns true when Codex auth state exists and the Codex SDK package can be resolved', async () => {
+    fs.writeFileSync(path.join(codexHome, 'auth.json'), '{"ok":true}');
+
+    const { hasCodexDefaultCredentials } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    expect(hasCodexDefaultCredentials()).toBe(true);
+  });
+
+  it('returns false when Codex auth state exists but the Codex SDK package cannot be resolved', async () => {
+    fs.writeFileSync(path.join(codexHome, 'auth.json'), '{"ok":true}');
+    mockResolvePackageEntryPoint.mockReturnValue(null);
+
+    const { hasCodexDefaultCredentials } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    expect(hasCodexDefaultCredentials()).toBe(false);
+  });
+
+  it('returns false when neither CODEX_API_KEY nor Codex auth state exists', async () => {
+    const { hasCodexDefaultCredentials } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    expect(hasCodexDefaultCredentials()).toBe(false);
+  });
+
+  it('creates reusable Codex text and web-search providers with a read-only sandbox', async () => {
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const providers = getCodexDefaultProviders();
+    const cachedProviders = getCodexDefaultProviders();
+
+    expect(cachedProviders).toBe(providers);
+    expect(providers.gradingProvider.id()).toBe('openai:codex-sdk');
+    expect(providers.gradingJsonProvider.id()).toBe('openai:codex-sdk');
+    expect(providers.llmRubricProvider?.id()).toBe('openai:codex-sdk');
+    expect(providers.suggestionsProvider).toBe(providers.gradingProvider);
+    expect(providers.synthesizeProvider).toBe(providers.gradingProvider);
+    expect(providers.gradingProvider.config).toMatchObject({
+      approval_policy: 'never',
+      cli_env: {
+        CODEX_HOME: codexHome,
+      },
+      sandbox_mode: 'read-only',
+      skip_git_repo_check: true,
+    });
+    expect(providers.gradingProvider.config.working_dir).toMatch(/promptfoo-codex-default-/);
+    expect(providers.gradingProvider.config.working_dir).not.toBe(
+      path.join(os.tmpdir(), 'promptfoo-codex-default'),
+    );
+    expect(providers.gradingJsonProvider.config.working_dir).toBe(
+      providers.gradingProvider.config.working_dir,
+    );
+    expect(providers.gradingJsonProvider.config.output_schema).toEqual({
+      type: 'object',
+      properties: {
+        pass: {
+          type: 'boolean',
+        },
+        score: {
+          type: 'number',
+        },
+        reason: {
+          type: 'string',
+        },
+      },
+      required: ['pass', 'score', 'reason'],
+      additionalProperties: false,
+    });
+    expect(providers.webSearchProvider?.config).toMatchObject({
+      network_access_enabled: true,
+      working_dir: providers.gradingProvider.config.working_dir,
+      web_search_mode: 'live',
+    });
+  });
+
+  it('isolates cached default providers by process API credential without using raw keys in cache keys', async () => {
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    mockProcessEnv({ CODEX_API_KEY: 'first-codex-key' });
+    const firstProviders = getCodexDefaultProviders();
+    expect((firstProviders.gradingProvider as OpenAICodexSDKProvider).apiKey).toBe(
+      'first-codex-key',
+    );
+
+    mockProcessEnv({ CODEX_API_KEY: 'second-codex-key' });
+    const secondProviders = getCodexDefaultProviders();
+
+    expect(secondProviders).not.toBe(firstProviders);
+    expect((secondProviders.gradingProvider as OpenAICodexSDKProvider).apiKey).toBe(
+      'second-codex-key',
+    );
+    expect((secondProviders.gradingProvider as OpenAICodexSDKProvider).getApiKey()).toBe(
+      'second-codex-key',
+    );
+  });
+
+  it('isolates cached default providers by process OpenAI API credential', async () => {
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    mockProcessEnv({ OPENAI_API_KEY: 'first-openai-key' });
+    const firstProviders = getCodexDefaultProviders();
+    expect((firstProviders.gradingProvider as OpenAICodexSDKProvider).getApiKey()).toBe(
+      'first-openai-key',
+    );
+
+    mockProcessEnv({ OPENAI_API_KEY: 'second-openai-key' });
+    const secondProviders = getCodexDefaultProviders();
+
+    expect(secondProviders).not.toBe(firstProviders);
+    expect((secondProviders.gradingProvider as OpenAICodexSDKProvider).getApiKey()).toBe(
+      'second-openai-key',
+    );
+  });
+
+  it('isolates cached default providers by env override API credential', async () => {
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const firstProviders = getCodexDefaultProviders({ CODEX_API_KEY: 'first-codex-key' });
+    const firstProvidersAgain = getCodexDefaultProviders({ CODEX_API_KEY: 'first-codex-key' });
+    const secondProviders = getCodexDefaultProviders({ CODEX_API_KEY: 'second-codex-key' });
+
+    expect(firstProvidersAgain).toBe(firstProviders);
+    expect(secondProviders).not.toBe(firstProviders);
+    expect((firstProviders.gradingProvider as OpenAICodexSDKProvider).getApiKey()).toBe(
+      'first-codex-key',
+    );
+    expect((secondProviders.gradingProvider as OpenAICodexSDKProvider).getApiKey()).toBe(
+      'second-codex-key',
+    );
+  });
+
+  it('evicts and shuts down idle least-recently-used cached providers when credentials rotate', async () => {
+    vi.useFakeTimers();
+
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const firstProviders = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-key-0' });
+    const firstGradingShutdown = vi.spyOn(
+      firstProviders.gradingProvider as OpenAICodexSDKProvider,
+      'shutdown',
+    );
+    const firstGradingJsonShutdown = vi.spyOn(
+      firstProviders.gradingJsonProvider as OpenAICodexSDKProvider,
+      'shutdown',
+    );
+    const firstWebSearchShutdown = vi.spyOn(
+      firstProviders.webSearchProvider as OpenAICodexSDKProvider,
+      'shutdown',
+    );
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    expect(firstGradingShutdown).not.toHaveBeenCalled();
+    expect(firstGradingJsonShutdown).not.toHaveBeenCalled();
+    expect(firstWebSearchShutdown).not.toHaveBeenCalled();
+
+    await vi.runOnlyPendingTimersAsync();
+
+    expect(firstGradingShutdown).toHaveBeenCalledTimes(1);
+    expect(firstGradingJsonShutdown).toHaveBeenCalledTimes(1);
+    expect(firstWebSearchShutdown).toHaveBeenCalledTimes(1);
+
+    const recachedFirstProviders = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-key-0' });
+    expect(recachedFirstProviders).not.toBe(firstProviders);
+  });
+
+  it('defers evicted provider shutdown until in-flight calls finish', async () => {
+    vi.useFakeTimers();
+
+    const { OpenAICodexSDKProvider } = await import('../../../src/providers/openai/codex-sdk');
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const inFlightCall = createDeferred<any>();
+    const callApiSpy = vi
+      .spyOn(OpenAICodexSDKProvider.prototype, 'callApi')
+      .mockImplementation(() => inFlightCall.promise);
+
+    const firstProviders = getCodexDefaultProviders({
+      CODEX_API_KEY: 'codex-key-0',
+    }) as unknown as CodexProviderBundle;
+    const shutdowns = spyAndStubShutdowns(firstProviders);
+
+    const resultPromise = firstProviders.gradingProvider.callApi('test prompt');
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    expect(shutdowns.grading).not.toHaveBeenCalled();
+    expect(shutdowns.gradingJson).not.toHaveBeenCalled();
+    expect(shutdowns.webSearch).not.toHaveBeenCalled();
+
+    inFlightCall.resolve({ output: 'ok' });
+    await expect(resultPromise).resolves.toEqual({ output: 'ok' });
+
+    expect(shutdowns.grading).not.toHaveBeenCalled();
+    expect(shutdowns.gradingJson).not.toHaveBeenCalled();
+    expect(shutdowns.webSearch).not.toHaveBeenCalled();
+
+    await vi.runOnlyPendingTimersAsync();
+
+    expect(shutdowns.grading).toHaveBeenCalledTimes(1);
+    expect(shutdowns.gradingJson).toHaveBeenCalledTimes(1);
+    expect(shutdowns.webSearch).toHaveBeenCalledTimes(1);
+    expect(callApiSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('keeps evicted providers usable for sequential calls before the eviction grace period elapses', async () => {
+    vi.useFakeTimers();
+
+    const { OpenAICodexSDKProvider } = await import('../../../src/providers/openai/codex-sdk');
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const callApiSpy = vi
+      .spyOn(OpenAICodexSDKProvider.prototype, 'callApi')
+      .mockResolvedValue({ output: 'ok' });
+
+    const firstProviders = getCodexDefaultProviders({
+      CODEX_API_KEY: 'codex-key-0',
+    }) as unknown as CodexProviderBundle;
+    const shutdowns = spyAndStubShutdowns(firstProviders);
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    await expect(firstProviders.gradingProvider.callApi('first prompt')).resolves.toEqual({
+      output: 'ok',
+    });
+    await expect(firstProviders.gradingProvider.callApi('second prompt')).resolves.toEqual({
+      output: 'ok',
+    });
+
+    expect(shutdowns.grading).not.toHaveBeenCalled();
+    expect(shutdowns.gradingJson).not.toHaveBeenCalled();
+    expect(shutdowns.webSearch).not.toHaveBeenCalled();
+
+    await vi.runOnlyPendingTimersAsync();
+
+    expect(shutdowns.grading).toHaveBeenCalledTimes(1);
+    expect(shutdowns.gradingJson).toHaveBeenCalledTimes(1);
+    expect(shutdowns.webSearch).toHaveBeenCalledTimes(1);
+    expect(callApiSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it('returns true when only OPENAI_API_KEY is set, mirroring the provider api-key resolution', async () => {
+    mockProcessEnv({ CODEX_API_KEY: undefined, OPENAI_API_KEY: 'openai-only-key' });
+
+    const { hasCodexDefaultCredentials } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    expect(hasCodexDefaultCredentials()).toBe(true);
+  });
+
+  it('resurrects providers that received a callApi after eviction shutdown completed', async () => {
+    vi.useFakeTimers();
+
+    const { OpenAICodexSDKProvider } = await import('../../../src/providers/openai/codex-sdk');
+    const { providerRegistry: registry } = await import('../../../src/providers/providerRegistry');
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const callApiSpy = vi
+      .spyOn(OpenAICodexSDKProvider.prototype, 'callApi')
+      .mockResolvedValue({ output: 'ok' });
+    const registerSpy = vi.spyOn(registry, 'register');
+
+    const firstProviders = getCodexDefaultProviders({
+      CODEX_API_KEY: 'codex-key-0',
+    }) as unknown as CodexProviderBundle;
+    const shutdowns = spyAndStubShutdowns(firstProviders);
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    // Drain the grace timer so eviction shutdown actually fires (mocked) before we
+    // attempt to use the provider again.
+    await vi.runOnlyPendingTimersAsync();
+    expect(shutdowns.grading).toHaveBeenCalledTimes(1);
+    expect(shutdowns.gradingJson).toHaveBeenCalledTimes(1);
+    expect(shutdowns.webSearch).toHaveBeenCalledTimes(1);
+
+    registerSpy.mockClear();
+
+    // Holder still has firstProviders.gradingProvider — the wrapped callApi must wait
+    // for the in-flight shutdown to settle, re-register the providers, and forward the
+    // call rather than returning a zombie response.
+    await expect(firstProviders.gradingProvider.callApi('post-shutdown prompt')).resolves.toEqual({
+      output: 'ok',
+    });
+
+    expect(callApiSpy).toHaveBeenCalledWith('post-shutdown prompt');
+    // Re-registration covers the unique providers (grading, gradingJson, webSearch).
+    expect(registerSpy).toHaveBeenCalledTimes(3);
+    const reregistered = new Set<unknown>(registerSpy.mock.calls.map((call) => call[0]));
+    expect(reregistered.has(firstProviders.gradingProvider)).toBe(true);
+    expect(reregistered.has(firstProviders.gradingJsonProvider)).toBe(true);
+    expect(reregistered.has(firstProviders.webSearchProvider)).toBe(true);
+  });
+
+  it('does not re-arm an eviction timer after clearCodexDefaultProvidersForTesting', async () => {
+    vi.useFakeTimers();
+
+    const { OpenAICodexSDKProvider } = await import('../../../src/providers/openai/codex-sdk');
+    const { clearCodexDefaultProvidersForTesting, getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    const inFlightCall = createDeferred<any>();
+    vi.spyOn(OpenAICodexSDKProvider.prototype, 'callApi').mockImplementation(
+      () => inFlightCall.promise,
+    );
+
+    const firstProviders = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-key-0' });
+    const firstGradingShutdown = vi
+      .spyOn(firstProviders.gradingProvider as OpenAICodexSDKProvider, 'shutdown')
+      .mockResolvedValue(undefined);
+
+    // Start a call that will keep activeCalls > 0 across the eviction.
+    const inFlightPromise = firstProviders.gradingProvider.callApi('long prompt');
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    // Cleanup before the in-flight call completes — this is the brittle window.
+    clearCodexDefaultProvidersForTesting();
+
+    // Resolve the in-flight call. The wrapper finally must NOT re-arm a shutdown timer.
+    inFlightCall.resolve({ output: 'late' });
+    await expect(inFlightPromise).resolves.toEqual({ output: 'late' });
+
+    // No pending timers — confirms scheduleCodexDefaultProviderBundleShutdown's
+    // cancelled guard kept the finally block from re-arming a timer.
+    expect(vi.getTimerCount()).toBe(0);
+    await vi.runOnlyPendingTimersAsync();
+    expect(firstGradingShutdown).not.toHaveBeenCalled();
+  });
+
+  it('partitions cache by the resolved api key, not by every raw credential slot', async () => {
+    // OPENAI_API_KEY takes precedence in OpenAICodexSDKProvider.getApiKey(), so two
+    // env-overrides that differ only in the (ignored) CODEX_API_KEY fallback resolve
+    // to the same effective credential and must hit the same cached bundle.
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    mockProcessEnv({ OPENAI_API_KEY: 'shared-openai-key' });
+
+    const first = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-A' });
+    const second = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-B' });
+    expect(second).toBe(first);
+
+    // Rotating the OPENAI key (the resolved credential) does invalidate the cache.
+    mockProcessEnv({ OPENAI_API_KEY: 'rotated-openai-key' });
+    const third = getCodexDefaultProviders({ CODEX_API_KEY: 'codex-A' });
+    expect(third).not.toBe(first);
+  });
+
+  it('keeps resurrected bundles bounded by re-entering eviction-pending after their next idle', async () => {
+    // Without this, a held reference that survives eviction would live indefinitely
+    // outside both the LRU map and the eviction set, defeating the cache-size bound for
+    // long-running processes.
+    vi.useFakeTimers();
+
+    const { OpenAICodexSDKProvider } = await import('../../../src/providers/openai/codex-sdk');
+    const { getCodexDefaultProviders } = await import(
+      '../../../src/providers/openai/codexDefaults'
+    );
+
+    vi.spyOn(OpenAICodexSDKProvider.prototype, 'callApi').mockResolvedValue({ output: 'ok' });
+
+    const firstProviders = getCodexDefaultProviders({
+      CODEX_API_KEY: 'codex-key-0',
+    }) as unknown as CodexProviderBundle;
+    const shutdowns = spyAndStubShutdowns(firstProviders);
+
+    fillCacheToOverflow(getCodexDefaultProviders);
+
+    // First eviction shutdown.
+    await vi.runOnlyPendingTimersAsync();
+    expect(shutdowns.grading).toHaveBeenCalledTimes(1);
+
+    // Resurrection via held reference.
+    await firstProviders.gradingProvider.callApi('post-shutdown prompt');
+
+    // After the call finishes, the wrapper's finally must have re-armed a shutdown
+    // timer; running it should fire shutdown a second time on the same bundle, proving
+    // the resurrected bundle did not escape the cleanup path.
+    await vi.runOnlyPendingTimersAsync();
+    expect(shutdowns.grading).toHaveBeenCalledTimes(2);
+    expect(shutdowns.gradingJson).toHaveBeenCalledTimes(2);
+    expect(shutdowns.webSearch).toHaveBeenCalledTimes(2);
+  });
+});

test/providers/openai/completion.test.ts

@@ -0,0 +1,255 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { disableCache, enableCache, fetchWithCache } from '../../../src/cache';
+import logger from '../../../src/logger';
+import { OpenAiCompletionProvider } from '../../../src/providers/openai/completion';
+import { mockProcessEnv } from '../../util/utils';
+import { getOpenAiMissingApiKeyMessage, restoreEnvVar } from './shared';
+
+vi.mock('../../../src/cache');
+vi.mock('../../../src/logger');
+
+const mockFetchWithCache = vi.mocked(fetchWithCache);
+
+describe('OpenAI Provider', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    disableCache();
+    // Set a default API key for tests unless explicitly testing missing key
+    mockProcessEnv({ OPENAI_API_KEY: 'test-api-key' });
+  });
+
+  afterEach(() => {
+    enableCache();
+  });
+
+  describe('OpenAiCompletionProvider', () => {
+    const mockResponse = {
+      data: {
+        choices: [{ text: 'Test output' }],
+        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+      severity: 'info',
+    };
+
+    it('should call API successfully with text completion', async () => {
+      mockFetchWithCache.mockResolvedValue(mockResponse);
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(mockFetchWithCache).toHaveBeenCalledTimes(1);
+      expect(result.output).toBe('Test output');
+      expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, numRequests: 1 });
+    });
+
+    it('should handle API errors', async () => {
+      mockFetchWithCache.mockResolvedValue({
+        data: {
+          error: {
+            message: 'Test error',
+            type: 'test_error',
+          },
+        },
+        cached: false,
+        status: 400,
+        statusText: 'Bad Request',
+      });
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.error).toBeDefined();
+      expect(result.error).toContain('Test error');
+    });
+
+    it('should handle fetch errors', async () => {
+      mockFetchWithCache.mockRejectedValue(new Error('Network error'));
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.error).toBeDefined();
+      expect(result.error).toContain('Network error');
+    });
+
+    it('should handle missing API key', async () => {
+      // Save the original env var and clear it for this test
+      const originalApiKey = process.env.OPENAI_API_KEY;
+      mockProcessEnv({ OPENAI_API_KEY: undefined });
+
+      try {
+        const provider = new OpenAiCompletionProvider('text-davinci-003', {
+          config: {
+            apiKeyRequired: true,
+          },
+          env: {
+            OPENAI_API_KEY: undefined,
+          },
+        });
+
+        await expect(provider.callApi('Test prompt')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage(),
+        );
+      } finally {
+        restoreEnvVar('OPENAI_API_KEY', originalApiKey);
+      }
+    });
+
+    it('should use custom apiKeyEnvar in missing API key errors', async () => {
+      const originalApiKey = process.env.OPENAI_API_KEY;
+      const originalCustomApiKey = process.env.CUSTOM_OPENAI_KEY;
+      mockProcessEnv({ OPENAI_API_KEY: undefined });
+      mockProcessEnv({ CUSTOM_OPENAI_KEY: undefined });
+
+      try {
+        const provider = new OpenAiCompletionProvider('text-davinci-003', {
+          config: {
+            apiKeyEnvar: 'CUSTOM_OPENAI_KEY',
+          },
+          env: {
+            OPENAI_API_KEY: undefined,
+            CUSTOM_OPENAI_KEY: undefined,
+          },
+        });
+
+        await expect(provider.callApi('Test prompt')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage('CUSTOM_OPENAI_KEY'),
+        );
+      } finally {
+        restoreEnvVar('OPENAI_API_KEY', originalApiKey);
+        restoreEnvVar('CUSTOM_OPENAI_KEY', originalCustomApiKey);
+      }
+    });
+
+    it('should warn about unknown model', () => {
+      const warnSpy = vi.spyOn(logger, 'warn');
+
+      new OpenAiCompletionProvider('unknown-model');
+
+      expect(warnSpy).toHaveBeenCalledWith(
+        'FYI: Using unknown OpenAI completion model: unknown-model',
+      );
+      warnSpy.mockRestore();
+    });
+
+    it('should handle cached responses', async () => {
+      mockFetchWithCache.mockResolvedValue({
+        ...mockResponse,
+        cached: true,
+      });
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.cached).toBe(true);
+      expect(result.output).toBe('Test output');
+    });
+
+    it('should handle responses without usage information', async () => {
+      mockFetchWithCache.mockResolvedValue({
+        data: {
+          choices: [{ text: 'Test output' }],
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toBe('Test output');
+      expect(result.tokenUsage).toEqual({});
+    });
+
+    it('should handle fetchWithCache returning undefined response', async () => {
+      mockFetchWithCache.mockResolvedValue(undefined as any);
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(mockFetchWithCache).toHaveBeenCalledTimes(1);
+      expect(result.error).toContain('Cannot destructure property');
+    });
+
+    it('should pass custom headers from config', async () => {
+      mockFetchWithCache.mockResolvedValue(mockResponse);
+
+      const customHeaders = {
+        'X-Test-Header': 'test-value',
+      };
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003', {
+        config: {
+          headers: customHeaders,
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      expect(mockFetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'Content-Type': 'application/json',
+            'X-OpenAI-Originator': 'promptfoo',
+            'X-Test-Header': 'test-value',
+          }),
+        }),
+        expect.any(Number),
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it('should pass passthrough config fields in body', async () => {
+      mockFetchWithCache.mockResolvedValue(mockResponse);
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003', {
+        config: {
+          passthrough: { logprobs: 3 },
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const actualCall = mockFetchWithCache.mock.calls[0];
+      const body = JSON.parse(actualCall[1]?.body as string);
+      expect(body.logprobs).toBe(3);
+    });
+
+    it('should handle response parsing errors', async () => {
+      mockFetchWithCache.mockResolvedValue({
+        data: {}, // Missing choices array
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003');
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.error).toMatch(/API error:/);
+    });
+
+    it('should handle invalid OPENAI_STOP env var', async () => {
+      mockProcessEnv({ OPENAI_STOP: '{invalid json}' });
+
+      const provider = new OpenAiCompletionProvider('text-davinci-003', {
+        config: {
+          apiKey: 'test-api-key',
+        },
+      });
+
+      await expect(provider.callApi('test')).rejects.toThrow(
+        /OPENAI_STOP is not a valid JSON string/,
+      );
+
+      mockProcessEnv({ OPENAI_STOP: undefined });
+    });
+  });
+});

test/providers/openai/defaults.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest';
+import {
+  DefaultEmbeddingProvider,
+  DefaultGradingJsonProvider,
+  DefaultGradingProvider,
+  DefaultModerationProvider,
+  DefaultSuggestionsProvider,
+  DefaultWebSearchProvider,
+} from '../../../src/providers/openai/defaults';
+
+describe('OpenAI default providers', () => {
+  describe('DefaultEmbeddingProvider', () => {
+    it('should use correct model version', () => {
+      expect(DefaultEmbeddingProvider.modelName).toBe('text-embedding-3-large');
+      expect(DefaultEmbeddingProvider.id()).toBe('openai:text-embedding-3-large');
+    });
+  });
+
+  describe('DefaultGradingProvider', () => {
+    it('should use correct model version and configuration', () => {
+      expect(DefaultGradingProvider.modelName).toBe('gpt-5.5-2026-04-23');
+      expect(DefaultGradingProvider.id()).toBe('openai:gpt-5.5-2026-04-23');
+      expect(DefaultGradingProvider.config).toEqual({});
+    });
+  });
+
+  describe('DefaultGradingJsonProvider', () => {
+    it('should use correct model version and JSON configuration', () => {
+      expect(DefaultGradingJsonProvider.modelName).toBe('gpt-5.5-2026-04-23');
+      expect(DefaultGradingJsonProvider.id()).toBe('openai:gpt-5.5-2026-04-23');
+      expect(DefaultGradingJsonProvider.config).toEqual({
+        response_format: { type: 'json_object' },
+      });
+    });
+  });
+
+  describe('DefaultSuggestionsProvider', () => {
+    it('should use correct model version', () => {
+      expect(DefaultSuggestionsProvider.modelName).toBe('gpt-5.5-2026-04-23');
+      expect(DefaultSuggestionsProvider.id()).toBe('openai:gpt-5.5-2026-04-23');
+      expect(DefaultSuggestionsProvider.config).toEqual({});
+    });
+  });
+
+  describe('DefaultModerationProvider', () => {
+    it('should use correct model version', () => {
+      expect(DefaultModerationProvider.modelName).toBe('omni-moderation-latest');
+      expect(DefaultModerationProvider.id()).toBe('openai:omni-moderation-latest');
+    });
+  });
+
+  describe('DefaultWebSearchProvider', () => {
+    it('should use correct model snapshot and web search configuration', () => {
+      expect(DefaultWebSearchProvider.modelName).toBe('gpt-5.5-2026-04-23');
+      expect(DefaultWebSearchProvider.id()).toBe('openai:gpt-5.5-2026-04-23');
+      expect(DefaultWebSearchProvider.config).toEqual({
+        tools: [{ type: 'web_search_preview' }],
+      });
+    });
+  });
+});

test/providers/openai/embedding.test.ts

@@ -0,0 +1,158 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { disableCache, enableCache, fetchWithCache } from '../../../src/cache';
+import { OpenAiEmbeddingProvider } from '../../../src/providers/openai/embedding';
+import { mockProcessEnv } from '../../util/utils';
+import { getOpenAiMissingApiKeyMessage } from './shared';
+
+vi.mock('../../../src/cache');
+
+describe('OpenAI Provider', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    disableCache();
+  });
+
+  afterEach(() => {
+    enableCache();
+  });
+
+  describe('OpenAiEmbeddingProvider', () => {
+    const configuredEmbeddingCostPerToken = 0.42 / 1e6;
+    const provider = new OpenAiEmbeddingProvider('text-embedding-3-large', {
+      config: {
+        apiKey: 'test-key',
+        cost: configuredEmbeddingCostPerToken,
+      },
+    });
+
+    it('should call embedding API successfully', async () => {
+      const mockResponse = {
+        data: [
+          {
+            embedding: [0.1, 0.2, 0.3],
+          },
+        ],
+        usage: {
+          total_tokens: 10,
+          prompt_tokens: 0,
+          completion_tokens: 0,
+        },
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: mockResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await provider.callEmbeddingApi('test text');
+      const expectedCost = 10 * provider.config.cost!;
+      expect(result.embedding).toEqual([0.1, 0.2, 0.3]);
+      expect(result.tokenUsage).toEqual({
+        total: 10,
+        prompt: 0,
+        completion: 0,
+        numRequests: 1,
+      });
+      expect(result.cost).toBeCloseTo(expectedCost, 12);
+    });
+
+    it('should pass through embedding request fields', async () => {
+      const passthroughProvider = new OpenAiEmbeddingProvider('text-embedding-3-small', {
+        config: {
+          apiKey: 'test-key',
+          passthrough: {
+            dimensions: 8,
+            encoding_format: 'float',
+          },
+        },
+      });
+
+      const mockEmbeddingResponse = {
+        data: [{ embedding: [0.1, 0.2, 0.3] }],
+        usage: {
+          total_tokens: 10,
+          prompt_tokens: 10,
+          completion_tokens: 0,
+        },
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: mockEmbeddingResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      await passthroughProvider.callEmbeddingApi('test text');
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.stringContaining('/embeddings'),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'X-OpenAI-Originator': 'promptfoo',
+          }),
+          body: JSON.stringify({
+            input: 'test text',
+            model: 'text-embedding-3-small',
+            dimensions: 8,
+            encoding_format: 'float',
+          }),
+        }),
+        expect.any(Number),
+        'json',
+        false,
+        undefined,
+      );
+      expect(mockEmbeddingResponse.usage.completion_tokens).toBe(0);
+    });
+
+    it('should handle API errors', async () => {
+      vi.mocked(fetchWithCache).mockRejectedValue(new Error('API error'));
+
+      const result = await provider.callEmbeddingApi('test text');
+      expect(result.error).toBe('API call error: Error: API error');
+      expect(result.embedding).toBeUndefined();
+    });
+
+    it('should validate input type', async () => {
+      const result = await provider.callEmbeddingApi({ message: 'test' } as unknown as string);
+      expect(result.error).toBe(
+        'Invalid input type for embedding API. Expected string, got object. Input: {"message":"test"}',
+      );
+      expect(result.embedding).toBeUndefined();
+    });
+
+    it('should handle HTTP error status', async () => {
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: { error: { message: 'Unauthorized' } },
+        cached: false,
+        status: 401,
+        statusText: 'Unauthorized',
+      });
+
+      const result = await provider.callEmbeddingApi('test text');
+      expect(result.error).toBe(
+        'API error: 401 Unauthorized\n{"error":{"message":"Unauthorized"}}',
+      );
+      expect(result.embedding).toBeUndefined();
+    });
+
+    it('should validate API key', async () => {
+      const restoreEnv = mockProcessEnv({ OPENAI_API_KEY: undefined });
+
+      try {
+        const providerNoKey = new OpenAiEmbeddingProvider('text-embedding-3-large', {
+          config: {},
+        });
+
+        const result = await providerNoKey.callEmbeddingApi('test text');
+        expect(result.error).toBe(getOpenAiMissingApiKeyMessage());
+        expect(result.embedding).toBeUndefined();
+      } finally {
+        restoreEnv();
+      }
+    });
+  });
+});

test/providers/openai/image.functions.test.ts

@@ -0,0 +1,519 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { fetchWithCache } from '../../../src/cache';
+import {
+  buildStructuredImageOutputs,
+  calculateImageCost,
+  callOpenAiImageApi,
+  DALLE2_COSTS,
+  DALLE3_COSTS,
+  formatOutput,
+  GPT_IMAGE2_COSTS,
+  prepareRequestBody,
+  processApiResponse,
+  validateSizeForModel,
+} from '../../../src/providers/openai/image';
+
+vi.mock('../../../src/cache', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    fetchWithCache: vi.fn(),
+  };
+});
+
+describe('OpenAI Image Provider Functions', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('validateSizeForModel', () => {
+    it('should validate valid DALL-E 3 sizes', () => {
+      expect(validateSizeForModel('1024x1024', 'dall-e-3')).toEqual({ valid: true });
+      expect(validateSizeForModel('1792x1024', 'dall-e-3')).toEqual({ valid: true });
+      expect(validateSizeForModel('1024x1792', 'dall-e-3')).toEqual({ valid: true });
+    });
+
+    it('should invalidate incorrect DALL-E 3 sizes', () => {
+      const result = validateSizeForModel('512x512', 'dall-e-3');
+      expect(result.valid).toBe(false);
+      expect(result.message).toContain('Invalid size "512x512" for DALL-E 3');
+    });
+
+    it('should validate valid DALL-E 2 sizes', () => {
+      expect(validateSizeForModel('256x256', 'dall-e-2')).toEqual({ valid: true });
+      expect(validateSizeForModel('512x512', 'dall-e-2')).toEqual({ valid: true });
+      expect(validateSizeForModel('1024x1024', 'dall-e-2')).toEqual({ valid: true });
+    });
+
+    it('should invalidate incorrect DALL-E 2 sizes', () => {
+      const result = validateSizeForModel('1792x1024', 'dall-e-2');
+      expect(result.valid).toBe(false);
+      expect(result.message).toContain('Invalid size "1792x1024" for DALL-E 2');
+    });
+
+    it('should validate any size for unknown models', () => {
+      expect(validateSizeForModel('any-size', 'unknown-model')).toEqual({ valid: true });
+    });
+
+    it('should validate GPT Image 2 sizes using dimensional constraints', () => {
+      expect(validateSizeForModel('auto', 'gpt-image-2')).toEqual({ valid: true });
+      expect(validateSizeForModel('1024x1024', 'gpt-image-2')).toEqual({ valid: true });
+      expect(validateSizeForModel('2048x1152', 'gpt-image-2')).toEqual({ valid: true });
+      expect(validateSizeForModel('3840x2160', 'gpt-image-2')).toEqual({ valid: true });
+    });
+
+    it('should invalidate GPT Image 2 sizes that violate constraints', () => {
+      expect(validateSizeForModel('512x512', 'gpt-image-2')).toMatchObject({
+        valid: false,
+      });
+      expect(validateSizeForModel('1024x1000', 'gpt-image-2')).toMatchObject({
+        valid: false,
+      });
+      expect(validateSizeForModel('3840x1024', 'gpt-image-2')).toMatchObject({
+        valid: false,
+      });
+      expect(validateSizeForModel('4096x2048', 'gpt-image-2').message).toContain(
+        'Invalid size "4096x2048" for GPT Image 2',
+      );
+    });
+  });
+
+  describe('formatOutput', () => {
+    it('should format URL output correctly', () => {
+      const data = {
+        data: [{ url: 'https://example.com/image.png' }],
+      };
+      const prompt = 'A test prompt';
+      const result = formatOutput(data, prompt, 'url');
+      expect(typeof result).toBe('string');
+      expect(result).toContain('![');
+      expect(result).toContain('](https://example.com/image.png)');
+    });
+
+    it('should sanitize prompt text with special characters', () => {
+      const data = {
+        data: [{ url: 'https://example.com/image.png' }],
+      };
+      const prompt = 'A test [with] brackets\nand newlines';
+      const result = formatOutput(data, prompt, 'url');
+      expect(typeof result).toBe('string');
+      expect(result).toContain('A test (with) brackets and newlines');
+    });
+
+    it('should format base64 output correctly', () => {
+      const mockData = {
+        data: [{ b64_json: 'base64encodeddata' }],
+      };
+      const result = formatOutput(mockData, 'prompt', 'b64_json');
+      expect(typeof result).toBe('string');
+      expect(result).toBe('data:image/png;base64,base64encodeddata');
+    });
+
+    it('should honor output format when formatting base64 output', () => {
+      const mockData = {
+        data: [{ b64_json: 'base64encodeddata' }],
+      };
+      const result = formatOutput(mockData, 'prompt', 'b64_json', 'jpeg');
+      expect(typeof result).toBe('string');
+      expect(result).toBe('data:image/jpeg;base64,base64encodeddata');
+    });
+
+    it('should return error when URL is missing', () => {
+      const data = { data: [{}] };
+      const result = formatOutput(data, 'prompt');
+      expect(typeof result).toBe('object');
+      expect(result).toHaveProperty('error');
+    });
+
+    it('should return error when base64 data is missing', () => {
+      const data = { data: [{}] };
+      const result = formatOutput(data, 'prompt', 'b64_json');
+      expect(typeof result).toBe('object');
+      expect(result).toHaveProperty('error');
+    });
+  });
+
+  describe('prepareRequestBody', () => {
+    it('should prepare basic request body correctly', () => {
+      const model = 'dall-e-2';
+      const prompt = 'A test prompt';
+      const size = '512x512';
+      const responseFormat = 'url';
+      const config = {};
+
+      const body = prepareRequestBody(model, prompt, size, responseFormat, config);
+
+      expect(body).toEqual({
+        model,
+        prompt,
+        size,
+        n: 1,
+        response_format: responseFormat,
+      });
+    });
+
+    it('should include n parameter from config', () => {
+      const config = { n: 2 };
+      const body = prepareRequestBody('dall-e-2', 'prompt', '512x512', 'url', config);
+      expect(body.n).toBe(2);
+    });
+
+    it('should include user parameter from config', () => {
+      const config = { user: 'promptfoo-user-123' };
+      const body = prepareRequestBody('gpt-image-2', 'prompt', '1024x1024', 'b64_json', config);
+      expect(body.user).toBe('promptfoo-user-123');
+    });
+
+    it('should include DALL-E 3 specific parameters', () => {
+      const config = {
+        quality: 'hd',
+        style: 'vivid',
+      };
+      const body = prepareRequestBody('dall-e-3', 'prompt', '1024x1024', 'url', config);
+
+      expect(body).toEqual({
+        model: 'dall-e-3',
+        prompt: 'prompt',
+        size: '1024x1024',
+        n: 1,
+        response_format: 'url',
+        quality: 'hd',
+        style: 'vivid',
+      });
+    });
+
+    it('should not include DALL-E 3 parameters for DALL-E 2', () => {
+      const config = {
+        quality: 'hd',
+        style: 'vivid',
+      };
+      const body = prepareRequestBody('dall-e-2', 'prompt', '512x512', 'url', config);
+
+      expect(body).not.toHaveProperty('quality');
+      expect(body).not.toHaveProperty('style');
+    });
+
+    it('should prepare GPT Image 2 request body without response_format', () => {
+      const config = {
+        quality: 'high',
+        background: 'opaque',
+        output_format: 'webp',
+        output_compression: 90,
+        moderation: 'low',
+      };
+      const body = prepareRequestBody('gpt-image-2', 'prompt', '2048x1152', 'url', config);
+
+      expect(body).toEqual({
+        model: 'gpt-image-2',
+        prompt: 'prompt',
+        size: '2048x1152',
+        n: 1,
+        quality: 'high',
+        background: 'opaque',
+        output_format: 'webp',
+        output_compression: 90,
+        moderation: 'low',
+      });
+    });
+  });
+
+  describe('calculateImageCost', () => {
+    it('should calculate correct cost for DALL-E 2', () => {
+      expect(calculateImageCost('dall-e-2', '256x256')).toBe(DALLE2_COSTS['256x256']);
+      expect(calculateImageCost('dall-e-2', '512x512')).toBe(DALLE2_COSTS['512x512']);
+      expect(calculateImageCost('dall-e-2', '1024x1024')).toBe(DALLE2_COSTS['1024x1024']);
+    });
+
+    it('should use default size cost if size is invalid for DALL-E 2', () => {
+      expect(calculateImageCost('dall-e-2', 'invalid-size')).toBe(DALLE2_COSTS['1024x1024']);
+    });
+
+    it('should calculate correct cost for standard DALL-E 3', () => {
+      expect(calculateImageCost('dall-e-3', '1024x1024', 'standard')).toBe(
+        DALLE3_COSTS['standard_1024x1024'],
+      );
+      expect(calculateImageCost('dall-e-3', '1024x1792', 'standard')).toBe(
+        DALLE3_COSTS['standard_1024x1792'],
+      );
+    });
+
+    it('should calculate correct cost for HD DALL-E 3', () => {
+      expect(calculateImageCost('dall-e-3', '1024x1024', 'hd')).toBe(DALLE3_COSTS['hd_1024x1024']);
+      expect(calculateImageCost('dall-e-3', '1024x1792', 'hd')).toBe(DALLE3_COSTS['hd_1024x1792']);
+    });
+
+    it('should use standard quality if quality is not specified for DALL-E 3', () => {
+      expect(calculateImageCost('dall-e-3', '1024x1024')).toBe(DALLE3_COSTS['standard_1024x1024']);
+    });
+
+    it('should use default cost if model is unknown', () => {
+      expect(calculateImageCost('unknown-model', '1024x1024')).toBe(0.04);
+    });
+
+    it('should multiply cost by number of images', () => {
+      expect(calculateImageCost('dall-e-2', '256x256', undefined, 3)).toBe(
+        DALLE2_COSTS['256x256'] * 3,
+      );
+      expect(calculateImageCost('dall-e-3', '1024x1024', 'standard', 2)).toBe(
+        DALLE3_COSTS['standard_1024x1024'] * 2,
+      );
+    });
+
+    it('should calculate correct cost for GPT Image 2 common sizes', () => {
+      expect(calculateImageCost('gpt-image-2', '1024x1024', 'low')).toBe(
+        GPT_IMAGE2_COSTS['low_1024x1024'],
+      );
+      expect(calculateImageCost('gpt-image-2', '1024x1536', 'medium')).toBe(
+        GPT_IMAGE2_COSTS['medium_1024x1536'],
+      );
+      expect(calculateImageCost('gpt-image-2', '1536x1024', 'high', 2)).toBe(
+        GPT_IMAGE2_COSTS['high_1536x1024'] * 2,
+      );
+    });
+
+    it('should not invent GPT Image 2 cost for auto quality or custom sizes', () => {
+      expect(calculateImageCost('gpt-image-2', '1024x1024')).toBeUndefined();
+      expect(calculateImageCost('gpt-image-2', '1024x1024', 'auto')).toBeUndefined();
+      expect(calculateImageCost('gpt-image-2', '2048x1152', 'high')).toBeUndefined();
+    });
+
+    it('should use default cost for models other than DALL-E 2 or 3', () => {
+      expect(calculateImageCost('gpt-4', '1024x1024')).toBe(0.04);
+      expect(calculateImageCost('', '1024x1024')).toBe(0.04);
+    });
+  });
+
+  describe('callOpenAiImageApi', () => {
+    it('should call fetchWithCache with correct parameters', async () => {
+      const mockResponse = {
+        data: { some: 'data' },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValue(mockResponse);
+
+      const url = 'https://api.openai.com/v1/images/generations';
+      const body = { model: 'dall-e-3', prompt: 'test' };
+      const headers = { 'Content-Type': 'application/json' };
+      const timeout = 30000;
+
+      const result = await callOpenAiImageApi(url, body, headers, timeout);
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        url,
+        {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(body),
+        },
+        timeout,
+      );
+      expect(result).toEqual(mockResponse);
+    });
+  });
+
+  describe('processApiResponse', () => {
+    it('should handle error in data', async () => {
+      const mockDeleteFromCache = vi.fn();
+      const data = {
+        error: { message: 'Some API error' },
+        deleteFromCache: mockDeleteFromCache,
+      };
+
+      const result = await processApiResponse(
+        data,
+        'prompt',
+        'url',
+        false,
+        'dall-e-2',
+        '512x512',
+        undefined,
+      );
+
+      expect(mockDeleteFromCache).toHaveBeenCalledWith();
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('Some API error');
+    });
+
+    it('should return formatted output for successful response', async () => {
+      const data = {
+        data: [{ url: 'https://example.com/image.png' }],
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'url',
+        false,
+        'dall-e-2',
+        '512x512',
+        undefined,
+      );
+
+      expect(result).toHaveProperty('output');
+      expect(result).toHaveProperty('cost');
+      expect(result.cost).toBe(DALLE2_COSTS['512x512']);
+    });
+
+    it('should include base64 flags for b64_json response format', async () => {
+      const data = {
+        data: [{ b64_json: 'base64data' }],
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'b64_json',
+        false,
+        'dall-e-3',
+        '1024x1024',
+        undefined,
+        'standard',
+      );
+
+      expect(result).toHaveProperty('isBase64', true);
+      expect(result).toHaveProperty('format', 'json');
+    });
+
+    it('should use output_format when building structured base64 images', async () => {
+      const data = {
+        data: [{ b64_json: 'base64data' }],
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'b64_json',
+        false,
+        'gpt-image-1',
+        '1024x1024',
+        undefined,
+        'low',
+        1,
+        'webp',
+      );
+
+      expect(result).toMatchObject({
+        output: 'data:image/webp;base64,base64data',
+        images: [{ data: 'data:image/webp;base64,base64data', mimeType: 'image/webp' }],
+      });
+    });
+
+    it('should set cost to 0 for cached responses', async () => {
+      const data = {
+        data: [{ url: 'https://example.com/image.png' }],
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'url',
+        true,
+        'dall-e-2',
+        '512x512',
+        undefined,
+      );
+
+      expect(result.cost).toBe(0);
+    });
+
+    it('should map image API usage to token usage and metadata', async () => {
+      const data = {
+        data: [{ b64_json: 'base64data' }],
+        usage: {
+          total_tokens: 30,
+          input_tokens: 10,
+          output_tokens: 20,
+          input_tokens_details: { text_tokens: 10, image_tokens: 0 },
+        },
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'b64_json',
+        false,
+        'gpt-image-2',
+        '1024x1024',
+        undefined,
+      );
+
+      expect(result).toMatchObject({
+        tokenUsage: {
+          prompt: 10,
+          completion: 20,
+          total: 30,
+          numRequests: 1,
+        },
+        metadata: {
+          usage: data.usage,
+        },
+      });
+      expect(result.cost).toBeCloseTo((10 * 5 + 20 * 30) / 1e6, 12);
+    });
+
+    it('should handle errors during output formatting', async () => {
+      const mockDeleteFromCache = vi.fn();
+      const data = {
+        data: undefined,
+        deleteFromCache: mockDeleteFromCache,
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'url',
+        false,
+        'dall-e-2',
+        '512x512',
+        undefined,
+      );
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('API error: TypeError');
+      expect(result.error).toContain('Cannot read properties of undefined');
+      expect(mockDeleteFromCache).toHaveBeenCalledWith();
+    });
+
+    it('should handle a specific error case with malformed response', async () => {
+      const mockDeleteFromCache = vi.fn();
+      const data = {
+        data: { data: 'not-an-array' },
+        deleteFromCache: mockDeleteFromCache,
+      };
+
+      const result = await processApiResponse(
+        data,
+        'test prompt',
+        'url',
+        false,
+        'dall-e-2',
+        '512x512',
+        undefined,
+      );
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('API error:');
+      expect(mockDeleteFromCache).toHaveBeenCalledWith();
+    });
+  });
+
+  describe('buildStructuredImageOutputs', () => {
+    it('should infer mime type from URL extensions', () => {
+      expect(
+        buildStructuredImageOutputs({
+          data: [{ url: 'https://example.com/image.jpg?size=large' }],
+        }),
+      ).toEqual([{ data: 'https://example.com/image.jpg?size=large', mimeType: 'image/jpeg' }]);
+    });
+
+    it('should omit mime type when URL extension is unknown', () => {
+      expect(
+        buildStructuredImageOutputs({
+          data: [{ url: 'https://example.com/generated-image' }],
+        }),
+      ).toEqual([{ data: 'https://example.com/generated-image' }]);
+    });
+  });
+});

test/providers/openai/index.test.ts

@@ -0,0 +1,154 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import {
+  DEFAULT_OPENAI_ORIGINATOR,
+  OPENAI_ORIGINATOR_HEADER,
+  OpenAiGenericProvider,
+} from '../../../src/providers/openai/index';
+import { mockProcessEnv } from '../../util/utils';
+
+describe('OpenAI Provider', () => {
+  describe('OpenAiGenericProvider', () => {
+    const provider = new OpenAiGenericProvider('test-model', {
+      config: {
+        apiKey: 'test-key',
+        organization: 'test-org',
+      },
+    });
+
+    beforeEach(() => {
+      mockProcessEnv({}, { clear: true });
+    });
+
+    it('should generate correct API URL', () => {
+      expect(provider.getApiUrl()).toBe('https://api.openai.com/v1');
+    });
+
+    it('should use custom API host', () => {
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiHost: 'custom.openai.com' },
+      });
+      expect(customProvider.getApiUrl()).toBe('https://custom.openai.com/v1');
+    });
+
+    it('should use custom API base URL', () => {
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiBaseUrl: 'https://custom.api.com/openai' },
+      });
+      expect(customProvider.getApiUrl()).toBe('https://custom.api.com/openai');
+    });
+
+    it('should get organization', () => {
+      expect(provider.getOrganization()).toBe('test-org');
+    });
+
+    it('should get organization from env', () => {
+      mockProcessEnv({ OPENAI_ORGANIZATION: 'env-org' });
+      const envProvider = new OpenAiGenericProvider('test-model');
+      expect(envProvider.getOrganization()).toBe('env-org');
+    });
+
+    it('should include the default originator header and allow explicit overrides', () => {
+      expect(provider.getOpenAiRequestHeaders()).toEqual({
+        [OPENAI_ORIGINATOR_HEADER]: DEFAULT_OPENAI_ORIGINATOR,
+        'OpenAI-Organization': 'test-org',
+      });
+
+      expect(
+        provider.getOpenAiRequestHeaders({
+          [OPENAI_ORIGINATOR_HEADER]: 'custom-originator',
+        }),
+      ).toMatchObject({
+        [OPENAI_ORIGINATOR_HEADER]: 'custom-originator',
+      });
+    });
+
+    // These two cases assert the FULL header object with toEqual on purpose: the bug
+    // being guarded is a *duplicate* case-variant header sneaking into the output, so
+    // the test must fail if any extra key (e.g. a second canonical-case header) appears.
+    // toMatchObject would allow such an extra key through and miss the regression.
+    it('should treat originator overrides case-insensitively', () => {
+      expect(
+        provider.getOpenAiRequestHeaders({
+          'x-openai-originator': 'custom-originator',
+        }),
+      ).toEqual({
+        'x-openai-originator': 'custom-originator',
+        'OpenAI-Organization': 'test-org',
+      });
+    });
+
+    it('should treat organization header overrides case-insensitively', () => {
+      expect(
+        provider.getOpenAiRequestHeaders({
+          'openai-organization': 'custom-org',
+        }),
+      ).toEqual({
+        'X-OpenAI-Originator': 'promptfoo',
+        'openai-organization': 'custom-org',
+      });
+    });
+
+    it('should not attribute compatible endpoints unless explicitly configured', () => {
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiBaseUrl: 'https://custom.api.com/openai' },
+      });
+
+      expect(customProvider.getOpenAiRequestHeaders()).not.toHaveProperty(OPENAI_ORIGINATOR_HEADER);
+      expect(
+        customProvider.getOpenAiRequestHeaders({
+          [OPENAI_ORIGINATOR_HEADER]: 'custom-originator',
+        }),
+      ).toMatchObject({
+        [OPENAI_ORIGINATOR_HEADER]: 'custom-originator',
+      });
+    });
+
+    it('should get API key', () => {
+      expect(provider.getApiKey()).toBe('test-key');
+    });
+
+    it('should get API key from env', () => {
+      mockProcessEnv({ OPENAI_API_KEY: 'env-key' });
+      const envProvider = new OpenAiGenericProvider('test-model');
+      expect(envProvider.getApiKey()).toBe('env-key');
+    });
+
+    it('should get API key from custom env var', () => {
+      mockProcessEnv({ CUSTOM_API_KEY: 'custom-key' });
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiKeyEnvar: 'CUSTOM_API_KEY' },
+      });
+      expect(customProvider.getApiKey()).toBe('custom-key');
+    });
+
+    it('should generate correct ID', () => {
+      expect(provider.id()).toBe('openai:test-model');
+    });
+
+    it('should generate custom ID with API host', () => {
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiHost: 'custom.openai.com' },
+      });
+      expect(customProvider.id()).toBe('test-model');
+    });
+
+    it('should have correct string representation', () => {
+      expect(provider.toString()).toBe('[OpenAI Provider test-model]');
+    });
+
+    it('should require API key by default', () => {
+      expect(provider.requiresApiKey()).toBe(true);
+    });
+
+    it('should allow disabling API key requirement', () => {
+      const customProvider = new OpenAiGenericProvider('test-model', {
+        config: { apiKeyRequired: false },
+      });
+      expect(customProvider.requiresApiKey()).toBe(false);
+    });
+
+    it('should throw not implemented for callApi', async () => {
+      await expect(provider.callApi('test prompt')).rejects.toThrow('Not implemented');
+    });
+  });
+});

test/providers/openai/moderation.test.ts

@@ -0,0 +1,994 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { fetchWithCache, getCache, getScopedCacheKey, isCacheEnabled } from '../../../src/cache';
+import {
+  formatModerationInput,
+  type ImageInput,
+  isImageInput,
+  isTextInput,
+  OpenAiModerationProvider,
+  supportsImageInput,
+  type TextInput,
+} from '../../../src/providers/openai/moderation';
+import { getOpenAiMissingApiKeyMessage } from './shared';
+
+vi.mock('../../../src/cache');
+vi.mock('../../../src/logger');
+
+describe('OpenAiModerationProvider', () => {
+  // Standard setup for all tests
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(isCacheEnabled).mockImplementation(function () {
+      return false;
+    });
+    vi.mocked(getScopedCacheKey).mockImplementation(function (cacheKey) {
+      return cacheKey;
+    });
+    vi.mocked(fetchWithCache).mockImplementation(async function () {
+      return {
+        data: {},
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      };
+    });
+  });
+
+  // Helper function to create a provider instance
+  const createProvider = (modelName = 'text-moderation-latest') => {
+    return new OpenAiModerationProvider(modelName, {
+      config: { apiKey: 'test-key' },
+    });
+  };
+
+  describe('Basic functionality', () => {
+    it('should moderate content and detect harmful content', async () => {
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [
+          {
+            flagged: true,
+            categories: {
+              hate: true,
+              'hate/threatening': false,
+            },
+            category_scores: {
+              hate: 0.99,
+              'hate/threatening': 0.01,
+            },
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+
+      expect(result).toEqual({
+        flags: [
+          {
+            code: 'hate',
+            description: 'hate',
+            confidence: 0.99,
+          },
+        ],
+      });
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.stringContaining('/moderations'),
+        expect.objectContaining({
+          method: 'POST',
+          headers: expect.objectContaining({
+            'Content-Type': 'application/json',
+            'x-promptfoo-silent': 'true',
+            Authorization: 'Bearer test-key',
+            'X-OpenAI-Originator': 'promptfoo',
+          }),
+          body: expect.stringContaining('"model":"text-moderation-latest"'),
+        }),
+        expect.any(Number),
+        'json',
+        true,
+        undefined,
+      );
+    });
+
+    it('should return empty flags for safe content', async () => {
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [
+          {
+            flagged: false,
+            categories: {
+              hate: false,
+              'hate/threatening': false,
+            },
+            category_scores: {
+              hate: 0.01,
+              'hate/threatening': 0.01,
+            },
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+
+      expect(result).toEqual({
+        flags: [],
+      });
+    });
+
+    it('should use flagged categories when scores are close together', async () => {
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [
+          {
+            flagged: true,
+            categories: {
+              hate: true,
+              'hate/threatening': false,
+            },
+            category_scores: {
+              hate: 0.5,
+              'hate/threatening': 0.49,
+            },
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+
+      expect(result).toEqual({
+        flags: [
+          {
+            code: 'hate',
+            description: 'hate',
+            confidence: 0.5,
+          },
+        ],
+      });
+    });
+  });
+
+  describe('Error handling', () => {
+    it('should handle API call errors', async () => {
+      const provider = createProvider();
+
+      vi.mocked(fetchWithCache).mockRejectedValueOnce(new Error('API Error'));
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+
+      expect(result).toEqual({
+        error: expect.stringContaining('API call error'),
+      });
+    });
+
+    it('should handle error responses from API', async () => {
+      const provider = createProvider();
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: { error: 'Invalid request' },
+        status: 400,
+        statusText: 'Bad Request',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+      expect(result).toEqual({
+        error: expect.stringContaining('API error: 400'),
+      });
+    });
+
+    it('should check for API key availability', async () => {
+      // Create provider with empty API key - instead of testing the throw,
+      // we'll mock getApiKey to return empty and verify handleApiError is used
+      const provider = createProvider();
+      vi.spyOn(provider, 'getApiKey').mockReturnValue('');
+
+      // Mock the logger to verify error is logged
+      const logger = (await import('../../../src/logger')).default;
+      const errorSpy = vi.spyOn(logger, 'error');
+
+      const result = await provider.callModerationApi('user', 'assistant');
+
+      // Verify we got an error response with the expected message
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain(getOpenAiMissingApiKeyMessage());
+      expect(errorSpy).toHaveBeenCalledWith(
+        'OpenAI moderation API error',
+        expect.objectContaining({
+          error: expect.stringContaining(getOpenAiMissingApiKeyMessage()),
+          hasData: false,
+        }),
+      );
+    });
+
+    it('should use custom apiKeyEnvar in missing API key errors', async () => {
+      const provider = new OpenAiModerationProvider('text-moderation-latest', {
+        config: {
+          apiKeyEnvar: 'CUSTOM_MODERATION_API_KEY',
+        },
+        env: {
+          OPENAI_API_KEY: undefined,
+          CUSTOM_MODERATION_API_KEY: undefined,
+        },
+      });
+      vi.spyOn(provider, 'getApiKey').mockReturnValue('');
+
+      const result = await provider.callModerationApi('user', 'assistant');
+
+      expect(result.error).toContain(getOpenAiMissingApiKeyMessage('CUSTOM_MODERATION_API_KEY'));
+    });
+
+    it('should handle empty results from API', async () => {
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [], // Empty results array
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user', 'assistant');
+      expect(result).toEqual({ flags: [] });
+    });
+
+    it('should handle non-empty results with no flagged categories', async () => {
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-124',
+        model: 'text-moderation-latest',
+        results: [
+          {
+            flagged: false,
+            categories: {
+              hate: false,
+              'hate/threatening': false,
+              harassment: false,
+              'harassment/threatening': false,
+              illicit: false,
+              'illicit/violent': false,
+              'self-harm': false,
+              'self-harm/intent': false,
+              'self-harm/instructions': false,
+              sexual: false,
+              'sexual/minors': false,
+              violence: false,
+              'violence/graphic': false,
+            },
+            category_scores: {
+              hate: 0,
+              'hate/threatening': 0,
+              harassment: 0,
+              'harassment/threatening': 0,
+              illicit: 0,
+              'illicit/violent': 0,
+              'self-harm': 0,
+              'self-harm/intent': 0,
+              'self-harm/instructions': 0,
+              sexual: 0,
+              'sexual/minors': 0,
+              violence: 0,
+              'violence/graphic': 0,
+            },
+            category_applied_input_types: {},
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const result = await provider.callModerationApi('user', 'assistant');
+      expect(result).toEqual({ flags: [] });
+    });
+  });
+
+  describe('Caching', () => {
+    it('should use cache when enabled', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const headerSecret = 'Bearer cache-secret-header';
+      const provider = new OpenAiModerationProvider('text-moderation-latest', {
+        config: { apiKey: 'test-key', headers: { Authorization: headerSecret } },
+      });
+
+      const mockResponse = {
+        flags: [
+          {
+            code: 'hate',
+            description: 'hate',
+            confidence: 0.9,
+          },
+        ],
+      };
+
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(JSON.stringify(mockResponse)),
+        set: vi.fn(),
+      };
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+
+      const result = await provider.callModerationApi('user input', 'assistant response');
+
+      expect(result).toEqual({ ...mockResponse, cached: true });
+      expect(result.cached).toBe(true);
+      const cacheKey = mockCache.get.mock.calls[0][0] as string;
+      expect(cacheKey).toMatch(
+        /^openai:moderation:text-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheKey).not.toContain('assistant response');
+      expect(cacheKey).not.toContain(headerSecret);
+      expect(cacheKey).not.toContain('test-key');
+      // Verify fetchWithCache wasn't called because cache was used
+      expect(fetchWithCache).not.toHaveBeenCalled();
+    });
+
+    it('should store results in cache when caching is enabled', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const provider = createProvider();
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [
+          {
+            flagged: true,
+            categories: { hate: true },
+            category_scores: { hate: 0.95 },
+          },
+        ],
+      };
+
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null), // No cached response
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await provider.callModerationApi('user', 'assistant');
+
+      // Verify we attempted to save to cache
+      const cacheKey = mockCache.set.mock.calls[0][0] as string;
+      expect(cacheKey).toMatch(
+        /^openai:moderation:text-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheKey).not.toContain('assistant');
+      expect(mockCache.set).toHaveBeenCalledWith(
+        cacheKey,
+        expect.stringContaining('{"flags":[{"code":"hate"'),
+      );
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        expect.any(Number),
+        'json',
+        true,
+        undefined,
+      );
+    });
+
+    it('should build cache keys from normalized moderation inputs', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const provider = createProvider('omni-moderation-latest');
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null),
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: {
+          id: 'modr-123',
+          model: 'omni-moderation-latest',
+          results: [{ flagged: false, categories: {}, category_scores: {} }],
+        },
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await provider.callModerationApi('user', 'same normalized text');
+      await provider.callModerationApi('user', [{ type: 'text', text: 'same normalized text' }]);
+
+      expect(mockCache.get.mock.calls[0][0]).toBe(mockCache.get.mock.calls[1][0]);
+    });
+
+    it('should isolate cache keys by resolved API key', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const providerA = new OpenAiModerationProvider('text-moderation-latest', {
+        config: { apiKey: 'sk-moderation-tenant-a' },
+      });
+      const providerB = new OpenAiModerationProvider('text-moderation-latest', {
+        config: { apiKey: 'sk-moderation-tenant-b' },
+      });
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null),
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: {
+          id: 'modr-123',
+          model: 'text-moderation-latest',
+          results: [{ flagged: false, categories: {}, category_scores: {} }],
+        },
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await providerA.callModerationApi('user', 'same sensitive text');
+      await providerB.callModerationApi('user', 'same sensitive text');
+
+      const [cacheKeyA, cacheKeyB] = mockCache.get.mock.calls.map(([key]) => key as string);
+      expect(cacheKeyA).toMatch(
+        /^openai:moderation:text-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheKeyB).toMatch(
+        /^openai:moderation:text-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheKeyA).not.toBe(cacheKeyB);
+      expect(cacheKeyA).not.toContain('same sensitive text');
+      expect(cacheKeyA).not.toContain('sk-moderation-tenant-a');
+      expect(cacheKeyB).not.toContain('sk-moderation-tenant-b');
+    });
+
+    it('should keep resolved API key cache identity stable across module reloads', async () => {
+      async function getCacheKeyFromFreshModule() {
+        vi.resetModules();
+        const freshCacheModule = await import('../../../src/cache');
+        const mockCache = {
+          get: vi.fn().mockResolvedValue(JSON.stringify({ flags: [] })),
+          set: vi.fn(),
+        };
+
+        vi.mocked(freshCacheModule.isCacheEnabled).mockImplementation(function () {
+          return true;
+        });
+        vi.mocked(freshCacheModule.getCache).mockImplementation(function () {
+          return mockCache as any;
+        });
+
+        const { OpenAiModerationProvider: FreshOpenAiModerationProvider } = await import(
+          '../../../src/providers/openai/moderation'
+        );
+        const provider = new FreshOpenAiModerationProvider('text-moderation-latest', {
+          config: { apiKey: 'sk-moderation-reload' },
+        });
+
+        await provider.callModerationApi('user', 'same sensitive text');
+
+        return mockCache.get.mock.calls[0][0] as string;
+      }
+
+      const cacheKeyA = await getCacheKeyFromFreshModule();
+      const cacheKeyB = await getCacheKeyFromFreshModule();
+
+      expect(cacheKeyA).toBe(cacheKeyB);
+      expect(cacheKeyA).toMatch(
+        /^openai:moderation:text-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheKeyA).not.toContain('same sensitive text');
+      expect(cacheKeyA).not.toContain('sk-moderation-reload');
+    });
+
+    it('should deduplicate concurrent moderation requests with identical cache identity', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const provider = createProvider();
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null),
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [{ flagged: false, categories: {}, category_scores: {} }],
+      };
+      let resolveFetch: (value: any) => void;
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      vi.mocked(fetchWithCache).mockReturnValueOnce(
+        new Promise<any>((resolve) => {
+          resolveFetch = resolve;
+        }) as ReturnType<typeof fetchWithCache>,
+      );
+
+      const first = provider.callModerationApi('user', 'same sensitive text');
+      const second = provider.callModerationApi('user', 'same sensitive text');
+
+      await vi.waitFor(() => {
+        expect(fetchWithCache).toHaveBeenCalledTimes(1);
+      });
+
+      resolveFetch!({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await expect(Promise.all([first, second])).resolves.toEqual([{ flags: [] }, { flags: [] }]);
+    });
+
+    it('should not deduplicate in-flight requests across cache namespaces', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const provider = createProvider();
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null),
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'text-moderation-latest',
+        results: [{ flagged: false, categories: {}, category_scores: {} }],
+      };
+      const resolvers: Array<(value: any) => void> = [];
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      let namespaceIndex = 0;
+      vi.mocked(getScopedCacheKey).mockImplementation(function (cacheKey) {
+        const namespace = namespaceIndex++ === 0 ? 'repeat-0' : 'repeat-1';
+        return `${namespace}:${cacheKey}`;
+      });
+      vi.mocked(fetchWithCache).mockImplementation(
+        () =>
+          new Promise<any>((resolve) => {
+            resolvers.push(resolve);
+          }) as ReturnType<typeof fetchWithCache>,
+      );
+
+      const first = provider.callModerationApi('user', 'same sensitive text');
+      const second = provider.callModerationApi('user', 'same sensitive text');
+
+      await vi.waitFor(() => {
+        expect(fetchWithCache).toHaveBeenCalledTimes(2);
+      });
+
+      for (const resolveFetch of resolvers) {
+        resolveFetch({
+          data: mockResponse,
+          status: 200,
+          statusText: 'OK',
+          cached: false,
+        });
+      }
+
+      await expect(Promise.all([first, second])).resolves.toEqual([{ flags: [] }, { flags: [] }]);
+    });
+  });
+
+  describe('Multi-modal support', () => {
+    it('should format inputs correctly for omni-moderation models', async () => {
+      const provider = createProvider('omni-moderation-latest');
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'omni-moderation-latest',
+        results: [{ flagged: false, categories: {}, category_scores: {} }],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await provider.callModerationApi('user input', 'assistant response');
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          body: expect.stringContaining('"type":"text"'),
+        }),
+        expect.any(Number),
+        'json',
+        true,
+        undefined,
+      );
+    });
+
+    it('should handle mixed text and image inputs', async () => {
+      const provider = createProvider('omni-moderation-latest');
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'omni-moderation-latest',
+        results: [
+          {
+            flagged: true,
+            categories: {
+              violence: true,
+            },
+            category_scores: {
+              violence: 0.95,
+            },
+            category_applied_input_types: {
+              violence: ['image'],
+            },
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const imageInput: Array<TextInput | ImageInput> = [
+        { type: 'text' as const, text: 'Some text' },
+        {
+          type: 'image_url' as const,
+          image_url: { url: 'https://example.com/image.png' },
+        },
+      ];
+
+      const result = await provider.callModerationApi('user input', imageInput);
+
+      // Instead of checking for "applied to" text, just verify that we have flags
+      expect(result.flags!).toBeDefined();
+      expect(result.flags!.length).toBeGreaterThan(0);
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          body: expect.stringContaining('image_url'),
+        }),
+        expect.any(Number),
+        'json',
+        true,
+        undefined,
+      );
+    });
+
+    it('should hash cached mixed text and image inputs without leaking raw content', async () => {
+      vi.mocked(isCacheEnabled).mockImplementation(function () {
+        return true;
+      });
+
+      const apiKey = 'sk-secret-moderation-key';
+      const headerSecret = 'Bearer moderation-header-secret';
+      const textSecret = 'multimodal-secret-text';
+      const imageUrl = 'https://example.com/image.png?token=image-secret-token';
+      const provider = new OpenAiModerationProvider('omni-moderation-latest', {
+        config: {
+          apiKey,
+          headers: {
+            Authorization: headerSecret,
+          },
+        },
+      });
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'omni-moderation-latest',
+        results: [{ flagged: false, categories: {}, category_scores: {} }],
+      };
+
+      const mockCache = {
+        get: vi.fn().mockResolvedValue(null),
+        set: vi.fn().mockResolvedValue(undefined),
+      };
+
+      vi.mocked(getCache).mockImplementation(function () {
+        return mockCache as any;
+      });
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const imageInput: Array<TextInput | ImageInput> = [
+        { type: 'text' as const, text: textSecret },
+        {
+          type: 'image_url' as const,
+          image_url: { url: imageUrl },
+        },
+      ];
+
+      await provider.callModerationApi('user input', imageInput);
+
+      const cacheGetKey = mockCache.get.mock.calls[0][0] as string;
+      const cacheSetKey = mockCache.set.mock.calls[0][0] as string;
+      expect(cacheGetKey).toBe(cacheSetKey);
+      expect(cacheSetKey).toMatch(
+        /^openai:moderation:omni-moderation-latest:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
+      );
+      expect(cacheSetKey).not.toContain(textSecret);
+      expect(cacheSetKey).not.toContain(imageUrl);
+      expect(cacheSetKey).not.toContain('image-secret-token');
+      expect(cacheSetKey).not.toContain(apiKey);
+      expect(cacheSetKey).not.toContain(headerSecret);
+    });
+
+    it('should show which input types triggered each flag', async () => {
+      const provider = createProvider('omni-moderation-latest');
+
+      const mockResponse = {
+        id: 'modr-123',
+        model: 'omni-moderation-latest',
+        results: [
+          {
+            flagged: true,
+            categories: {
+              violence: true,
+              sexual: true,
+            },
+            category_scores: {
+              violence: 0.95,
+              sexual: 0.98,
+            },
+            category_applied_input_types: {
+              violence: ['image'],
+              sexual: ['text', 'image'],
+            },
+          },
+        ],
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: mockResponse,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const imageInput: Array<TextInput | ImageInput> = [
+        { type: 'text' as const, text: 'Some text' },
+        {
+          type: 'image_url' as const,
+          image_url: { url: 'https://example.com/image.png' },
+        },
+      ];
+
+      const result = await provider.callModerationApi('user input', imageInput);
+
+      const violenceFlag = result.flags?.find((f) => f.code === 'violence');
+      const sexualFlag = result.flags?.find((f) => f.code === 'sexual');
+
+      // Just verify that the flags exist with the correct codes
+      expect(violenceFlag).toBeDefined();
+      expect(sexualFlag).toBeDefined();
+    });
+  });
+
+  describe('Model configuration', () => {
+    it('should warn about unknown models', async () => {
+      // Import the mocked logger
+      const logger = (await import('../../../src/logger')).default;
+      const warnSpy = vi.spyOn(logger, 'warn');
+
+      new OpenAiModerationProvider('unknown-model', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(warnSpy).toHaveBeenCalledWith(
+        expect.stringContaining('unknown OpenAI moderation model'),
+      );
+    });
+
+    it('should accept custom API headers', async () => {
+      const provider = new OpenAiModerationProvider('text-moderation-latest', {
+        config: {
+          apiKey: 'test-key',
+          headers: {
+            'Custom-Header': 'custom-value',
+          },
+        },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValueOnce({
+        data: { id: 'modr-123', model: 'text-moderation-latest', results: [] },
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      await provider.callModerationApi('user', 'assistant');
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'Custom-Header': 'custom-value',
+            'x-promptfoo-silent': 'true',
+          }),
+        }),
+        expect.any(Number),
+        'json',
+        true,
+        undefined,
+      );
+    });
+  });
+});
+
+describe('Moderation Utility Functions', () => {
+  describe('Input Type Guards', () => {
+    it('should correctly identify text inputs', () => {
+      const textInput: TextInput = { type: 'text', text: 'test content' };
+      const imageInput: ImageInput = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      };
+
+      expect(isTextInput(textInput)).toBe(true);
+      expect(isTextInput(imageInput)).toBe(false);
+    });
+
+    it('should correctly identify image inputs', () => {
+      const textInput: TextInput = { type: 'text', text: 'test content' };
+      const imageInput: ImageInput = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      };
+
+      expect(isImageInput(imageInput)).toBe(true);
+      expect(isImageInput(textInput)).toBe(false);
+    });
+  });
+
+  describe('supportsImageInput', () => {
+    it('should return true for multi-modal models', () => {
+      expect(supportsImageInput('omni-moderation-latest')).toBe(true);
+      expect(supportsImageInput('omni-moderation-2024-09-26')).toBe(true);
+    });
+
+    it('should return false for text-only models', () => {
+      expect(supportsImageInput('text-moderation-latest')).toBe(false);
+      expect(supportsImageInput('text-moderation-stable')).toBe(false);
+      expect(supportsImageInput('text-moderation-007')).toBe(false);
+    });
+
+    it('should return false for unknown models', () => {
+      expect(supportsImageInput('nonexistent-model')).toBe(false);
+    });
+  });
+
+  describe('formatModerationInput', () => {
+    it('should return string as-is for text-only models', () => {
+      const input = 'test content';
+      expect(formatModerationInput(input, false)).toBe(input);
+    });
+
+    it('should wrap string in TextInput object for multi-modal models', () => {
+      const input = 'test content';
+      const expected = [{ type: 'text', text: input }];
+      expect(formatModerationInput(input, true)).toEqual(expected);
+    });
+
+    it('should return array inputs as-is for multi-modal models', () => {
+      const input = [
+        { type: 'text' as const, text: 'text content' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image.jpg' } },
+      ];
+      expect(formatModerationInput(input, true)).toEqual(input);
+    });
+
+    it('should filter out image inputs for text-only models', () => {
+      const input = [
+        { type: 'text' as const, text: 'text content' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image.jpg' } },
+      ];
+      expect(formatModerationInput(input, false)).toBe('text content');
+    });
+
+    it('should join multiple text inputs when filtering for text-only models', () => {
+      const input = [
+        { type: 'text' as const, text: 'first text' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image.jpg' } },
+        { type: 'text' as const, text: 'second text' },
+      ];
+      expect(formatModerationInput(input, false)).toBe('first text second text');
+    });
+
+    it('should handle empty text strings when joining for text-only models', () => {
+      const input = [
+        { type: 'text' as const, text: '' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image.jpg' } },
+        { type: 'text' as const, text: 'second text' },
+      ];
+      expect(formatModerationInput(input, false)).toBe(' second text');
+    });
+
+    it('should preserve leading and trailing whitespace in text inputs when joining', () => {
+      const input = [
+        { type: 'text' as const, text: '  first text  ' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image.jpg' } },
+        { type: 'text' as const, text: 'second text' },
+      ];
+      expect(formatModerationInput(input, false)).toBe('  first text   second text');
+    });
+
+    it('should return single text input unchanged when mixed with images for text-only models', () => {
+      const input = [
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image-1.jpg' } },
+        { type: 'text' as const, text: 'only text' },
+        { type: 'image_url' as const, image_url: { url: 'https://example.com/image-2.jpg' } },
+      ];
+      expect(formatModerationInput(input, false)).toBe('only text');
+    });
+  });
+});

test/providers/openai/responses/azure.test.ts

@@ -0,0 +1,241 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+import { setOpenAiEnv } from './setup';
+
+describe('OpenAiResponsesProvider Azure custom deployments', () => {
+  describe('Azure custom deployment detection', () => {
+    const AZURE_BASE_URL = 'https://my-resource.openai.azure.com/openai/v1';
+    const AZURE_MODEL = 'my-company-gpt-54-prod';
+
+    function mockAzureSuccessResponse(): void {
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: {
+          id: 'resp_abc123',
+          status: 'completed',
+          model: AZURE_MODEL,
+          output: [
+            {
+              type: 'message',
+              role: 'assistant',
+              content: [{ type: 'output_text', text: 'Response from Azure custom deployment' }],
+            },
+          ],
+          usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+    }
+
+    function getRequestBody(): Record<string, any> {
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      return JSON.parse(reqOptions.body);
+    }
+
+    it('should include explicit reasoning and verbosity for Azure custom deployment names', async () => {
+      mockAzureSuccessResponse();
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          reasoning: { effort: 'medium' },
+          temperature: 0.7,
+          verbosity: 'low',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+      const body = getRequestBody();
+
+      expect(body.model).toBe(AZURE_MODEL);
+      expect(body.reasoning).toEqual({ effort: 'medium' });
+      expect(body.text).toMatchObject({ format: { type: 'text' }, verbosity: 'low' });
+      expect(body.temperature).toBeUndefined();
+    });
+
+    it('should include reasoning_effort for Azure custom deployment names without verbosity', async () => {
+      mockAzureSuccessResponse();
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          reasoning_effort: 'medium',
+          temperature: 0.7,
+        },
+      });
+
+      await provider.callApi('Test prompt');
+      const body = getRequestBody();
+
+      expect(body.model).toBe(AZURE_MODEL);
+      expect(body.reasoning).toEqual({ effort: 'medium' });
+      expect(body.text).toEqual({ format: { type: 'text' } });
+      expect(body.temperature).toBeUndefined();
+    });
+
+    it('should include verbosity for Azure custom deployment names without reasoning', async () => {
+      mockAzureSuccessResponse();
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          temperature: 0.7,
+          verbosity: 'low',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+      const body = getRequestBody();
+
+      expect(body.model).toBe(AZURE_MODEL);
+      expect(body.reasoning).toBeUndefined();
+      expect(body.text).toMatchObject({ format: { type: 'text' }, verbosity: 'low' });
+      expect(body.temperature).toBe(0.7);
+    });
+
+    it('should preserve temperature when reasoning_effort is none', async () => {
+      mockAzureSuccessResponse();
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          reasoning_effort: 'none',
+          temperature: 0.7,
+        },
+      });
+
+      await provider.callApi('Test prompt');
+      const body = getRequestBody();
+
+      expect(body.reasoning).toEqual({ effort: 'none' });
+      expect(body.temperature).toBe(0.7);
+    });
+
+    it('should detect Azure deployment via apiHost', async () => {
+      mockAzureSuccessResponse();
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiHost: 'my-resource.openai.azure.com',
+          reasoning_effort: 'medium',
+          temperature: 0.7,
+        },
+      });
+
+      await provider.callApi('Test prompt');
+      const body = getRequestBody();
+
+      expect(body.reasoning).toEqual({ effort: 'medium' });
+      expect(body.temperature).toBeUndefined();
+    });
+
+    it('should detect Azure deployment via OpenAI endpoint environment variables', async () => {
+      setOpenAiEnv({ OPENAI_API_BASE_URL: AZURE_BASE_URL });
+
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          reasoning_effort: 'medium',
+          temperature: 0.7,
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.reasoning).toEqual({ effort: 'medium' });
+      expect(body.temperature).toBeUndefined();
+    });
+
+    it('should detect Azure deployment via provider env overrides', async () => {
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          reasoning_effort: 'medium',
+          temperature: 0.7,
+        },
+        env: {
+          OPENAI_API_HOST: 'my-resource.openai.azure.com',
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.reasoning).toEqual({ effort: 'medium' });
+      expect(body.temperature).toBeUndefined();
+    });
+
+    it('should not trigger Azure reasoning detection for non-Azure hosts', async () => {
+      const provider = new OpenAiResponsesProvider('custom-model', {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: 'https://api.example.com/v1',
+          reasoning_effort: 'medium',
+          temperature: 0.5,
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.reasoning).toBeUndefined();
+      expect(body.temperature).toBe(0.5);
+    });
+
+    it('should not trigger Azure reasoning detection when openai.azure.com appears outside the host', async () => {
+      const provider = new OpenAiResponsesProvider('custom-model', {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: 'https://api.example.com/proxy/openai.azure.com/v1',
+          reasoning_effort: 'medium',
+          temperature: 0.5,
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.reasoning).toBeUndefined();
+      expect(body.temperature).toBe(0.5);
+    });
+
+    it('should merge reasoning object with reasoning_effort', async () => {
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          reasoning_effort: 'high',
+          reasoning: { summary: 'concise' },
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.reasoning).toEqual({ effort: 'high', summary: 'concise' });
+    });
+
+    it('should use correct max_output_tokens default for verbosity-only deployments', async () => {
+      const provider = new OpenAiResponsesProvider(AZURE_MODEL, {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: AZURE_BASE_URL,
+          verbosity: 'low',
+        },
+      });
+
+      const { body } = await provider.getOpenAiBody('Test prompt');
+
+      expect(body.max_output_tokens).toBe(1024);
+      expect(body.text).toMatchObject({ verbosity: 'low' });
+    });
+  });
+});

test/providers/openai/responses/callbacks.test.ts

@@ -0,0 +1,484 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider function callbacks', () => {
+  describe('Function Tool Callbacks', () => {
+    it('should execute function callbacks and return the result', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            name: 'addNumbers',
+            id: 'call_123',
+            arguments: '{"a": 5, "b": 6}',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add 5 and 6');
+
+      expect(result.error).toBeUndefined();
+      expect(result.output).toBe('11');
+    });
+
+    it('should handle multiple function calls including status updates', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            name: 'addNumbers',
+            id: 'call_123',
+            arguments: '{"a": 5, "b": 6}',
+          },
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'addNumbers',
+            arguments: '{}',
+            call_id: 'call_123',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add 5 and 6');
+
+      expect(result.error).toBeUndefined();
+      // With our fix, the first call returns "11" and second call returns JSON (due to empty args)
+      // They should be joined with newline
+      expect(result.output).toContain('11');
+      expect(result.output).toContain('function_call');
+    });
+
+    it('should fall back to raw function call when callback fails', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            name: 'addNumbers',
+            id: 'call_123',
+            arguments: 'invalid json',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args); // This will throw
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add numbers with invalid JSON');
+
+      expect(result.error).toBeUndefined();
+      const parsedOutput = JSON.parse(result.output);
+      expect(parsedOutput.type).toBe('function_call');
+      expect(parsedOutput.name).toBe('addNumbers');
+      expect(parsedOutput.arguments).toBe('invalid json');
+    });
+
+    it('should handle function callbacks in assistant message content', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'function_call',
+                name: 'multiplyNumbers',
+                arguments: '{"a": 3, "b": 4}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            multiplyNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a * b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Multiply 3 and 4');
+
+      expect(result.error).toBeUndefined();
+      expect(result.output).toBe('12');
+    });
+
+    it('should handle single function call with empty arguments and status completed', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'addNumbers',
+            arguments: '{}',
+            call_id: 'call_123',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add 5 and 6');
+
+      expect(result.error).toBeUndefined();
+      const parsedOutput = JSON.parse(result.output);
+      expect(parsedOutput.type).toBe('function_call');
+      expect(parsedOutput.name).toBe('addNumbers');
+      expect(parsedOutput.status).toBe('no_arguments_provided');
+      expect(parsedOutput.note).toContain('Consider using the correct Responses API tool format');
+    });
+
+    it('should handle function call with empty arguments but no status', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            name: 'addNumbers',
+            arguments: '{}',
+            call_id: 'call_123',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args || '{}');
+              return JSON.stringify((a || 0) + (b || 0));
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add numbers');
+
+      expect(result.error).toBeUndefined();
+      // Should execute callback since no status=completed, even with empty args
+      expect(result.output).toBe('0');
+    });
+
+    it('should handle function call with status completed but non-empty arguments', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'addNumbers',
+            arguments: '{"a": 10, "b": 15}',
+            call_id: 'call_123',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Add 10 and 15');
+
+      expect(result.error).toBeUndefined();
+      // Should execute callback since arguments are not empty
+      expect(result.output).toBe('25');
+    });
+
+    it('should handle multiple function calls with all empty arguments', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'addNumbers',
+            arguments: '{}',
+            call_id: 'call_123',
+          },
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'multiplyNumbers',
+            arguments: '{}',
+            call_id: 'call_456',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            addNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a + b);
+            },
+            multiplyNumbers: async (args: string) => {
+              const { a, b } = JSON.parse(args);
+              return JSON.stringify(a * b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Do math operations');
+
+      expect(result.error).toBeUndefined();
+      // Should contain both function call JSONs
+      expect(result.output).toContain('addNumbers');
+      expect(result.output).toContain('multiplyNumbers');
+      expect(result.output).toContain('no_arguments_provided');
+      // Should have newline separator
+      expect(result.output).toContain('\n');
+    });
+
+    it('should handle function call with empty string arguments', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            status: 'completed',
+            name: 'greetUser',
+            arguments: '',
+            call_id: 'call_123',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          functionToolCallbacks: {
+            greetUser: async (_args: string) => {
+              return 'Hello!';
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Greet the user');
+
+      expect(result.error).toBeUndefined();
+      // Empty string arguments should still execute the callback
+      expect(result.output).toBe('Hello!');
+    });
+
+    it('should handle tool callback integration test with expected result format', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'function_call',
+            name: 'addNumbers',
+            id: 'call_123',
+            arguments: '{"a": 5, "b": 6}',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          temperature: 0,
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'function',
+              function: {
+                name: 'addNumbers',
+                description: 'Add two numbers together',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    a: { type: 'number' },
+                    b: { type: 'number' },
+                  },
+                  required: ['a', 'b'],
+                },
+              },
+            },
+          ],
+          tool_choice: 'auto',
+          functionToolCallbacks: {
+            addNumbers: async (parametersJsonString: string) => {
+              const { a, b } = JSON.parse(parametersJsonString);
+              return JSON.stringify(a + b);
+            },
+          },
+        },
+      });
+
+      const result = await provider.callApi('Please add the following numbers together: 5 and 6');
+
+      expect(result.error).toBeUndefined();
+      expect(result.output).toBe('11');
+    });
+  });
+});

test/providers/openai/responses/errors.test.ts

@@ -0,0 +1,324 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import logger from '../../../../src/logger';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+import { getOpenAiMissingApiKeyMessage } from '../shared';
+
+describe('OpenAiResponsesProvider error handling', () => {
+  it('should handle JSON schema validation errors correctly', async () => {
+    const mockApiResponse = {
+      error: {
+        message: 'The response format is invalid. Cannot parse as JSON schema.',
+        type: 'invalid_response_format',
+        code: 'json_schema_validation_error',
+        param: 'response_format',
+      },
+      status: 400,
+      statusText: 'Bad Request',
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 400,
+      statusText: 'Bad Request',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        response_format: {
+          type: 'json_schema',
+          json_schema: {
+            name: 'InvalidSchema',
+            strict: true,
+            schema: {
+              type: 'object',
+              properties: {
+                result: { type: 'string' },
+              },
+              // The API will complain about something even though the schema is valid
+              required: ['missing_field'],
+              additionalProperties: false,
+            },
+          },
+        },
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Assert error is present
+    expect(result.error).toContain('json_schema_validation_error');
+  });
+
+  it('should handle API errors correctly', async () => {
+    // Setup mock for fetchWithCache to return an error
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        error: {
+          message: 'Invalid request',
+          type: 'invalid_request_error',
+          code: 'invalid_api_key',
+        },
+      },
+      cached: false,
+      status: 400,
+      statusText: 'Bad Request',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'invalid-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Assertions
+    expect(result.error).toContain('API error');
+    expect(result.output).toBeUndefined();
+  });
+
+  it('should throw error when API key is not set', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {},
+    });
+
+    vi.spyOn(provider, 'getApiKey').mockReturnValue(undefined);
+
+    await expect(provider.callApi('Test prompt')).rejects.toThrow(getOpenAiMissingApiKeyMessage());
+  });
+
+  it('should use custom apiKeyEnvar in missing API key errors', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKeyEnvar: 'CUSTOM_RESPONSES_API_KEY',
+      },
+    });
+
+    vi.spyOn(provider, 'getApiKey').mockReturnValue(undefined);
+
+    await expect(provider.callApi('Test prompt')).rejects.toThrow(
+      getOpenAiMissingApiKeyMessage('CUSTOM_RESPONSES_API_KEY'),
+    );
+  });
+
+  it('should handle error in API response data correctly', async () => {
+    const mockApiResponse = {
+      error: {
+        message: 'Content policy violation',
+        type: 'content_policy_violation',
+        code: 'content_filter',
+      },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 400,
+      statusText: 'Bad Request',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const result = await provider.callApi('Test prompt');
+
+    expect(result.error).toContain('content_policy_violation');
+  });
+
+  it('should handle missing output array correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      // No output array
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify error about missing output array
+    expect(result.error).toContain('Invalid response format: Missing output array');
+  });
+
+  it('should handle successful JSON response payloads correctly', async () => {
+    // Mock API response
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Test response',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Ensure we get output without error
+    const result = await provider.callApi('Test prompt');
+
+    // Verify the API was called successfully
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.stringContaining('/responses'),
+      expect.anything(),
+      expect.anything(),
+      'json',
+      undefined,
+      undefined,
+    );
+    expect(result.output).toBe('Test response');
+    expect(result.error).toBeUndefined();
+  });
+
+  it('should handle null content in message output', async () => {
+    // Mock API response with content that will cause error during processing
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: null, // Will cause TypeError when trying to iterate
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API - since there's no content, it should return an empty string
+    const result = await provider.callApi('Test prompt');
+
+    // Verify we get a result with empty output
+    expect(result.output).toBe('');
+    expect(result.raw).toEqual(mockApiResponse);
+
+    // Error is not set since this is treated as an empty response, not an error
+    expect(result.error).toBeUndefined();
+  });
+
+  it('should handle error processing results with non-array output', async () => {
+    // Setup mock for fetchWithCache to return data that will trigger a processing error
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: 'not-an-array', // This will cause an error when trying to process as an array
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // This should have returned an invalid format error
+    expect(result.error).toBeTruthy();
+    expect(result.error).toContain('Invalid response format');
+    expect(result.output).toBeUndefined();
+
+    // The implementation doesn't include raw data when format is invalid
+    // So we shouldn't test for it
+  });
+
+  // Test for lines 169-174: Testing when fetch throws an error (not just returns error status)
+
+  it('should handle network errors correctly', async () => {
+    // Setup mock to throw an error
+    vi.mocked(cache.fetchWithCache).mockRejectedValue(new Error('Network error'));
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify error is handled correctly
+    expect(result.error).toContain('API call error:');
+    expect(result.error).toContain('Network error');
+
+    // Expect logger to be called with the error
+    expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('Network error'));
+  });
+});

test/providers/openai/responses/mcp.test.ts

@@ -0,0 +1,472 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider MCP request handling', () => {
+  describe('MCP (Model Context Protocol) support', () => {
+    it('should include MCP tools in request body correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Response with MCP tools',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              require_approval: 'never',
+              allowed_tools: ['ask_question'],
+            },
+          ],
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.tools).toBeDefined();
+      expect(body.tools).toHaveLength(1);
+      expect(body.tools[0]).toEqual({
+        type: 'mcp',
+        server_label: 'deepwiki',
+        server_url: 'https://mcp.deepwiki.com/mcp',
+        require_approval: 'never',
+        allowed_tools: ['ask_question'],
+      });
+    });
+
+    it('should handle MCP tools with authentication headers', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Response with authenticated MCP tools',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'stripe',
+              server_url: 'https://mcp.stripe.com',
+              headers: {
+                Authorization: 'Bearer sk-test_123',
+              },
+              require_approval: 'never',
+            },
+          ],
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.tools[0].headers).toEqual({
+        Authorization: 'Bearer sk-test_123',
+      });
+    });
+
+    it('should handle MCP list tools response correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_list_tools',
+            id: 'mcpl_123',
+            server_label: 'deepwiki',
+            tools: [
+              {
+                name: 'ask_question',
+                input_schema: {
+                  type: 'object',
+                  properties: {
+                    question: { type: 'string' },
+                    repoName: { type: 'string' },
+                  },
+                  required: ['question', 'repoName'],
+                },
+              },
+            ],
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'I can help you search repositories.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              require_approval: 'never',
+            },
+          ],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toContain('MCP Tools from deepwiki');
+      expect(result.output).toContain('ask_question');
+      expect(result.output).toContain('I can help you search repositories.');
+    });
+
+    it('should handle MCP tool call response correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_call',
+            id: 'mcp_456',
+            server_label: 'deepwiki',
+            name: 'ask_question',
+            arguments:
+              '{"question":"What is MCP?","repoName":"modelcontextprotocol/modelcontextprotocol"}',
+            output:
+              'MCP (Model Context Protocol) is an open protocol that standardizes how applications provide tools and context to LLMs.',
+            error: null,
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Based on the search results, MCP is a protocol for LLM integration.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 25, output_tokens: 20, total_tokens: 45 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toContain('MCP Tool Result (ask_question)');
+      expect(result.output).toContain('MCP (Model Context Protocol) is an open protocol');
+      expect(result.output).toContain(
+        'Based on the search results, MCP is a protocol for LLM integration.',
+      );
+    });
+
+    it('should handle MCP tool call error correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_call',
+            id: 'mcp_456',
+            server_label: 'deepwiki',
+            name: 'ask_question',
+            arguments: '{"question":"Invalid query"}',
+            output: null,
+            error: 'Repository not found',
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'I encountered an error while searching.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toContain('MCP Tool Error (ask_question)');
+      expect(result.output).toContain('Repository not found');
+      expect(result.output).toContain('I encountered an error while searching.');
+    });
+
+    it('should handle MCP approval request correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_approval_request',
+            id: 'mcpr_789',
+            server_label: 'deepwiki',
+            name: 'ask_question',
+            arguments: '{"question":"What is the latest version?","repoName":"facebook/react"}',
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 5, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              // require_approval defaults to requiring approval
+            },
+          ],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      expect(result.output).toContain('MCP Approval Required for deepwiki.ask_question');
+      expect(result.output).toContain('facebook/react');
+    });
+
+    it('should handle mixed MCP and regular tools correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'I have access to both MCP and regular tools.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 30, output_tokens: 15, total_tokens: 45 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'function',
+              function: {
+                name: 'get_weather',
+                description: 'Get weather information',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    location: { type: 'string' },
+                  },
+                  required: ['location'],
+                },
+              },
+            },
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              require_approval: 'never',
+            },
+          ],
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.tools).toHaveLength(2);
+      expect(body.tools[0].type).toBe('function');
+      expect(body.tools[0].function.name).toBe('get_weather');
+      expect(body.tools[1].type).toBe('mcp');
+      expect(body.tools[1].server_label).toBe('deepwiki');
+    });
+
+    it('should handle MCP tool configuration with selective approval correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Response with selective approval MCP tools',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              require_approval: {
+                never: {
+                  tool_names: ['ask_question', 'read_wiki_structure'],
+                },
+              },
+              allowed_tools: ['ask_question', 'read_wiki_structure', 'search_repo'],
+            },
+          ],
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.tools).toBeDefined();
+      expect(body.tools).toHaveLength(1);
+      expect(body.tools[0]).toEqual({
+        type: 'mcp',
+        server_label: 'deepwiki',
+        server_url: 'https://mcp.deepwiki.com/mcp',
+        require_approval: {
+          never: {
+            tool_names: ['ask_question', 'read_wiki_structure'],
+          },
+        },
+        allowed_tools: ['ask_question', 'read_wiki_structure', 'search_repo'],
+      });
+    });
+  });
+});

test/providers/openai/responses/mcpAssertions.test.ts

@@ -0,0 +1,141 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider MCP assertions', () => {
+  describe('Enhanced OpenAI tools assertion with MCP support', () => {
+    it('should validate MCP tool success correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_call',
+            id: 'mcp_456',
+            server_label: 'deepwiki',
+            name: 'ask_question',
+            arguments: '{"question":"What is MCP?"}',
+            output: 'MCP is a protocol for LLM integration.',
+            error: null,
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Based on the search results, MCP is a protocol for LLM integration.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 25, output_tokens: 20, total_tokens: 45 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'mcp',
+              server_label: 'deepwiki',
+              server_url: 'https://mcp.deepwiki.com/mcp',
+              require_approval: 'never',
+            },
+          ],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      // The output should contain MCP Tool Result
+      expect(result.output).toContain('MCP Tool Result (ask_question)');
+
+      // Test the enhanced assertion
+      const { handleIsValidOpenAiToolsCall } = await import('../../../../src/assertions/openai');
+      const assertionResult = await handleIsValidOpenAiToolsCall({
+        assertion: { type: 'is-valid-openai-tools-call' },
+        output: result.output,
+        provider,
+        test: { vars: {} },
+      } as any);
+
+      expect(assertionResult.pass).toBe(true);
+      expect(assertionResult.reason).toContain('MCP tool call succeeded for ask_question');
+    });
+
+    it('should validate MCP tool error correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4.1',
+        output: [
+          {
+            type: 'mcp_call',
+            id: 'mcp_456',
+            server_label: 'deepwiki',
+            name: 'ask_question',
+            arguments: '{"question":"Invalid query"}',
+            output: null,
+            error: 'Repository not found',
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'I encountered an error while searching.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4.1', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      // The output should contain MCP Tool Error
+      expect(result.output).toContain('MCP Tool Error (ask_question)');
+
+      // Test the enhanced assertion
+      const { handleIsValidOpenAiToolsCall } = await import('../../../../src/assertions/openai');
+      const assertionResult = await handleIsValidOpenAiToolsCall({
+        assertion: { type: 'is-valid-openai-tools-call' },
+        output: result.output,
+        provider,
+        test: { vars: {} },
+      } as any);
+
+      expect(assertionResult.pass).toBe(false);
+      expect(assertionResult.reason).toContain(
+        'MCP tool call failed for ask_question: Repository not found',
+      );
+    });
+  });
+});

test/providers/openai/responses/metadata.test.ts

@@ -0,0 +1,165 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider HTTP metadata', () => {
+  it('should include HTTP metadata in response', async () => {
+    const mockHeaders = {
+      'content-type': 'application/json',
+      'x-request-id': 'test-request-123',
+      'x-litellm-model-group': 'gpt-4o',
+    };
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          id: 'msg_abc123',
+          status: 'completed',
+          role: 'assistant',
+          content: [{ type: 'output_text', text: 'Test response' }],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+      headers: mockHeaders,
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: { apiKey: 'test-key' },
+    });
+
+    const result = await provider.callApi('Test prompt');
+
+    expect(result.metadata).toBeDefined();
+    expect(result.metadata?.http).toBeDefined();
+    expect(result.metadata?.http?.status).toBe(200);
+    expect(result.metadata?.http?.statusText).toBe('OK');
+    expect(result.metadata?.http?.headers).toEqual(mockHeaders);
+  });
+
+  it('should include HTTP metadata in error response', async () => {
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: { error: { message: 'Rate limit exceeded' } },
+      cached: false,
+      status: 429,
+      statusText: 'Too Many Requests',
+      headers: { 'retry-after': '60' },
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: { apiKey: 'test-key' },
+    });
+
+    const result = await provider.callApi('Test prompt');
+
+    expect(result.error).toBeDefined();
+    expect(result.metadata?.http?.status).toBe(429);
+    expect(result.metadata?.http?.statusText).toBe('Too Many Requests');
+    expect(result.metadata?.http?.headers).toEqual({ 'retry-after': '60' });
+  });
+
+  it('should handle truncation information correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Truncated response',
+            },
+          ],
+        },
+      ],
+      truncation: {
+        tokens_truncated: 100,
+        tokens_remaining: 200,
+        token_limit: 4096,
+      },
+      usage: { input_tokens: 3896, output_tokens: 100, total_tokens: 3996 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const result = await provider.callApi('Very long prompt that would be truncated');
+
+    expect(result.raw).toHaveProperty('truncation');
+    expect(result.raw.truncation.tokens_truncated).toBe(100);
+  });
+
+  it('should handle streaming responses correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Streaming response',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        stream: true,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"stream":true'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+  });
+});

test/providers/openai/responses/models.test.ts

@@ -0,0 +1,169 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it } from 'vitest';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+const unsupportedAudioModels = [
+  'gpt-audio',
+  'gpt-audio-2025-08-28',
+  'gpt-audio-1.5',
+  'gpt-audio-mini',
+  'gpt-audio-mini-2025-12-15',
+  'gpt-audio-mini-2025-10-06',
+] as const;
+
+describe('OpenAiResponsesProvider model registry', () => {
+  it('should support various model names', async () => {
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4o');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1-2025-04-14');
+    // GPT-5 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.2-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.3-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-sol');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-terra');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-luna');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5-2026-04-23');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.5-pro-2026-04-23',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-2026-03-05');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-mini-2026-03-17',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-nano-2026-03-17',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-pro-2026-03-05',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-mini');
+    for (const model of unsupportedAudioModels) {
+      expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).not.toContain(model);
+    }
+    // GPT-4.5 models deprecated as of 2025-07-14, removed from API
+  });
+
+  it('should support the latest o-series reasoning models', async () => {
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-2025-04-16');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o4-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o4-mini-2025-04-16');
+  });
+
+  it('should support gpt-4.1 and its variants', async () => {
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1-2025-04-14');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-4.1-mini-2025-04-14',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4.1-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-4.1-nano-2025-04-14',
+    );
+  });
+
+  it('should support gpt-5 and its variants', async () => {
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5-mini');
+  });
+
+  it('should include all expected model names', async () => {
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4o');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4o-2024-08-06');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4o-2024-11-20');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-4o-2024-05-13');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-2024-12-17');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-preview');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-preview-2024-09-12');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-mini-2024-09-12');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o1-pro-2025-03-19');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-pro-2025-06-10');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-2025-04-16');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o4-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o4-mini-2025-04-16');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-mini-2025-01-31');
+    // GPT-4.5 models deprecated as of 2025-07-14, removed from API
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('codex-mini-latest');
+    // GPT-5.1 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.1');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.1-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.1-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.1-codex');
+    // GPT-5.2 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.2-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.2-codex');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.2-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.2-pro-2025-12-11',
+    );
+    // GPT-5.3 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.3-chat-latest');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.3-codex');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.3-codex-spark');
+    // GPT-5.6 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-sol');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-terra');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.6-luna');
+    // GPT-5.5 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5-2026-04-23');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.5-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.5-pro-2026-04-23',
+    );
+    // GPT-5.4 models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-2026-03-05');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-mini');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-mini-2026-03-17',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-nano');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-nano-2026-03-17',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('gpt-5.4-pro');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'gpt-5.4-pro-2026-03-05',
+    );
+    // Audio models are Chat Completions-only while Responses audio remains unsupported.
+    for (const model of unsupportedAudioModels) {
+      expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).not.toContain(model);
+    }
+    // Deep research models
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o3-deep-research');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'o3-deep-research-2025-06-26',
+    );
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain('o4-mini-deep-research');
+    expect(OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES).toContain(
+      'o4-mini-deep-research-2025-06-26',
+    );
+  });
+});

test/providers/openai/responses/reasoning.test.ts

@@ -0,0 +1,527 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+import { LONG_RUNNING_MODEL_TIMEOUT_MS } from '../../../../src/providers/shared';
+import { setOpenAiEnv } from './setup';
+import type { Mock } from 'vitest';
+
+describe('OpenAiResponsesProvider reasoning models', () => {
+  it('should prefer OPENAI_MAX_COMPLETION_TOKENS over OPENAI_MAX_TOKENS for reasoning models', async () => {
+    setOpenAiEnv({
+      OPENAI_MAX_COMPLETION_TOKENS: '4096',
+      OPENAI_MAX_TOKENS: '2048',
+    });
+
+    const provider = new OpenAiResponsesProvider('o1-preview', {
+      config: { apiKey: 'test-key' },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+    expect(body.max_output_tokens).toBe(4096);
+  });
+
+  it('should fall back to OPENAI_MAX_TOKENS for reasoning models when OPENAI_MAX_COMPLETION_TOKENS is unset', async () => {
+    setOpenAiEnv({ OPENAI_MAX_TOKENS: '2048' });
+
+    const provider = new OpenAiResponsesProvider('o1-preview', {
+      config: { apiKey: 'test-key' },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+    expect(body.max_output_tokens).toBe(2048);
+  });
+
+  it('should not apply a hardcoded max_output_tokens default for reasoning models', async () => {
+    const provider = new OpenAiResponsesProvider('o1-preview', {
+      config: { apiKey: 'test-key' },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+    expect(body.max_output_tokens).toBeUndefined();
+    expect('max_output_tokens' in body).toBe(false);
+  });
+
+  it('should handle reasoning models correctly', async () => {
+    // Mock API response for o1-pro model
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      object: 'response',
+      created_at: 1234567890,
+      status: 'completed',
+      model: 'o1-pro',
+      output: [
+        {
+          type: 'message',
+          id: 'msg_abc123',
+          status: 'completed',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'This is a response from o1-pro',
+            },
+          ],
+        },
+      ],
+      usage: {
+        input_tokens: 15,
+        output_tokens: 30,
+        output_tokens_details: {
+          reasoning_tokens: 100,
+        },
+        total_tokens: 45,
+      },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider with reasoning model settings
+    const provider = new OpenAiResponsesProvider('o1-pro', {
+      config: {
+        apiKey: 'test-key',
+        reasoning_effort: 'medium',
+        max_completion_tokens: 2000,
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify the request body includes reasoning effort
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"reasoning":{"effort":"medium"}'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+
+    // Assertions
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('This is a response from o1-pro');
+    // Just test that the total tokens is present, but don't test for reasoning tokens
+    // as the implementation may handle these details differently
+    expect(result.tokenUsage?.total).toBe(45);
+  });
+
+  it.each([
+    { model: 'gpt-5.6', reasoningEffort: 'max', maxOutputTokens: 4000 },
+    { model: 'gpt-5.6-sol', reasoningEffort: 'max', maxOutputTokens: 4000 },
+    { model: 'gpt-5.6-terra', reasoningEffort: 'max', maxOutputTokens: 4000 },
+    { model: 'gpt-5.6-luna', reasoningEffort: 'max', maxOutputTokens: 4000 },
+    { model: 'o3', reasoningEffort: 'high', maxOutputTokens: 2000 },
+    { model: 'o3-pro', reasoningEffort: 'high', maxOutputTokens: 2000 },
+    { model: 'o4-mini', reasoningEffort: 'medium', maxOutputTokens: 1000 },
+    { model: 'codex-mini-latest', reasoningEffort: 'medium', maxOutputTokens: 1000 },
+  ] as const)('should configure $model model correctly with reasoning parameters', async ({
+    model,
+    reasoningEffort,
+    maxOutputTokens,
+  }) => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model,
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: `Response from ${model} model`,
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider(model, {
+      config: {
+        apiKey: 'test-key',
+        reasoning_effort: reasoningEffort,
+        max_output_tokens: maxOutputTokens,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    expect(body.model).toBe(model);
+    expect(body.reasoning).toEqual({ effort: reasoningEffort });
+    expect(body.max_output_tokens).toBe(maxOutputTokens);
+    expect(body.temperature).toBeUndefined();
+  });
+
+  it('should forward GPT-5.6 persisted reasoning and Pro mode', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-5.6-sol', {
+      config: {
+        reasoning: { effort: 'max', context: 'all_turns', mode: 'pro' },
+      },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+
+    expect(body.reasoning).toEqual({ effort: 'max', context: 'all_turns', mode: 'pro' });
+  });
+
+  describe('deep research model validation', () => {
+    it('should require web_search_preview tool for deep research models', async () => {
+      const provider = new OpenAiResponsesProvider('o3-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'code_interpreter' } as any], // Missing web_search_preview
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+      expect(result.error).toContain('requires the web_search_preview tool');
+      expect(result.error).toContain('o3-deep-research');
+      expect(cache.fetchWithCache).not.toHaveBeenCalled();
+    });
+
+    it('should accept deep research models with web_search_preview tool', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Research complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider('o4-mini-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'web_search_preview' } as any],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+      expect(result.error).toBeUndefined();
+      expect(result.output).toBe('Research complete');
+    });
+
+    it('should require MCP tools to have require_approval: never for deep research', async () => {
+      const provider = new OpenAiResponsesProvider('o3-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            { type: 'web_search_preview' } as any,
+            {
+              type: 'mcp',
+              server_label: 'test_server',
+              server_url: 'http://test.com',
+              require_approval: 'auto' as any, // Should be 'never'
+            },
+          ],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+      expect(result.error).toContain("requires MCP tools to have require_approval: 'never'");
+      expect(cache.fetchWithCache).not.toHaveBeenCalled();
+    });
+
+    it('should use longer timeout for deep research models', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Research complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider('o3-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'web_search_preview' } as any],
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      // Check that fetchWithCache was called with 10-minute timeout
+      expect(cache.fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        LONG_RUNNING_MODEL_TIMEOUT_MS,
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it('should use longer timeout for gpt-5-pro models', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Response complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-5-pro', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      // Check that fetchWithCache was called with 10-minute timeout
+      expect(cache.fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        LONG_RUNNING_MODEL_TIMEOUT_MS,
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it('should use longer timeout for gpt-5.2-pro models', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Response complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-5.2-pro', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      expect(cache.fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        LONG_RUNNING_MODEL_TIMEOUT_MS,
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it.each([
+      'gpt-5.4-pro',
+      'gpt-5.5-pro',
+    ])('should use longer timeout for %s models', async (model) => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Response complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider(model, {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      expect(cache.fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        LONG_RUNNING_MODEL_TIMEOUT_MS,
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it('should use longer timeout for gpt-5.5-pro models', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Response complete' }],
+          },
+        ],
+        usage: { input_tokens: 100, output_tokens: 200 },
+      };
+
+      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
+        data: mockData,
+        status: 200,
+        statusText: 'OK',
+        cached: false,
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-5.5-pro', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      expect(cache.fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        LONG_RUNNING_MODEL_TIMEOUT_MS,
+        'json',
+        undefined,
+        undefined,
+      );
+    });
+
+    it('should handle reasoning items with empty summary correctly', async () => {
+      const mockData = {
+        output: [
+          {
+            type: 'reasoning',
+            summary: [], // Empty array (edge case)
+          },
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Final answer',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockData,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('o3-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'web_search_preview' } as any],
+        },
+      });
+
+      const result = await provider.callApi('Test prompt');
+
+      // Should only include the valid reasoning summary and final answer
+      expect(result.error).toBeUndefined();
+      expect(result.output).toBe('Final answer');
+      expect(result.output).not.toContain('Reasoning: []');
+    });
+  });
+
+  it('should handle reasoning items with summary correctly', async () => {
+    const mockData = {
+      output: [
+        {
+          type: 'reasoning',
+          summary: [{ type: 'summary_text', text: 'Valid reasoning summary' }],
+        },
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [{ type: 'output_text', text: 'Final answer' }],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockData,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const result = await provider.callApi('Test prompt');
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('Reasoning: Valid reasoning summary\nFinal answer');
+  });
+});

test/providers/openai/responses/refusals.test.ts

@@ -0,0 +1,151 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider refusals', () => {
+  describe('refusal handling', () => {
+    it('should handle explicit refusal content in message', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'refusal',
+                refusal: 'I cannot fulfill this request due to content policy violation.',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt with refusal');
+
+      expect(result.isRefusal).toBe(true);
+      expect(result.output).toBe('I cannot fulfill this request due to content policy violation.');
+    });
+
+    it('should handle direct refusal in message object', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            refusal: 'I cannot provide that information.',
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Test prompt with direct refusal');
+
+      expect(result.isRefusal).toBe(true);
+      expect(result.output).toBe('I cannot provide that information.');
+    });
+
+    it('should detect refusals in 400 API error with invalid_prompt code', async () => {
+      // Mock a 400 error response with invalid_prompt error code
+      const mockErrorResponse = {
+        data: {
+          error: {
+            message: 'some random error message',
+            type: 'invalid_request_error',
+            param: null,
+            code: 'invalid_prompt',
+          },
+        },
+        cached: false,
+        status: 400,
+        statusText: 'Bad Request',
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue(mockErrorResponse);
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('How do I create harmful content?');
+
+      // Should treat the error as a refusal output, not an error
+      expect(result.error).toBeUndefined();
+      expect(result.output).toContain('some random error message');
+      expect(result.output).toContain('400 Bad Request');
+      expect(result.isRefusal).toBe(true);
+    });
+
+    it('should still treat non-refusal 400 errors as errors', async () => {
+      // Mock a 400 error that is NOT a refusal (different error code)
+      const mockErrorResponse = {
+        data: {
+          error: {
+            message: "Invalid request: 'input' field is required",
+            type: 'invalid_request_error',
+            param: 'input',
+            code: 'missing_required_field',
+          },
+        },
+        cached: false,
+        status: 400,
+        statusText: 'Bad Request',
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue(mockErrorResponse);
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const result = await provider.callApi('Invalid request format');
+
+      // Should still be treated as an error since code is not invalid_prompt
+      expect(result.error).toBeDefined();
+      expect(result.error).toContain("'input' field is required");
+      expect(result.error).toContain('400 Bad Request');
+      expect(result.output).toBeUndefined();
+      expect(result.isRefusal).toBeUndefined();
+    });
+  });
+});

test/providers/openai/responses/request.test.ts

@@ -0,0 +1,524 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+import { setOpenAiEnv } from './setup';
+
+describe('OpenAiResponsesProvider request building', () => {
+  it('should format and call the responses API correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      object: 'response',
+      created_at: 1234567890,
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          id: 'msg_abc123',
+          status: 'completed',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'This is a test response',
+            },
+          ],
+        },
+      ],
+      usage: {
+        input_tokens: 10,
+        output_tokens: 20,
+        total_tokens: 30,
+      },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const result = await provider.callApi('Test prompt');
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.stringContaining('/responses'),
+      expect.objectContaining({
+        method: 'POST',
+        headers: expect.objectContaining({
+          'Content-Type': 'application/json',
+          Authorization: 'Bearer test-key',
+          'X-OpenAI-Originator': 'promptfoo',
+        }),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('This is a test response');
+    expect(result.tokenUsage?.total).toBe(30);
+  });
+
+  it('should handle system prompts correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Response with system prompt',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        instructions: 'You are a helpful assistant',
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"instructions":"You are a helpful assistant"'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+  });
+
+  it('should handle temperature and other parameters correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Response with custom parameters',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        temperature: 0.7,
+        top_p: 0.9,
+        max_completion_tokens: 1000,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    expect(body.temperature).toBe(0.7);
+    expect(body.top_p).toBe(0.9);
+    expect(body.max_output_tokens).toBeDefined();
+  });
+
+  it('should correctly send temperature: 0 in the request body', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [{ type: 'output_text', text: 'Response' }],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Test that temperature: 0 is correctly sent (not filtered out by falsy check)
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        temperature: 0,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    // temperature: 0 should be present in the request body
+    expect(body.temperature).toBe(0);
+    expect('temperature' in body).toBe(true);
+  });
+
+  it('should omit default temperature and max_output_tokens when omitDefaults is true', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        omitDefaults: true,
+      },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+
+    expect(body.temperature).toBeUndefined();
+    expect('temperature' in body).toBe(false);
+    expect(body.max_output_tokens).toBeUndefined();
+    expect('max_output_tokens' in body).toBe(false);
+  });
+
+  it('should use env defaults with omitDefaults when OPENAI env vars are set', async () => {
+    setOpenAiEnv({
+      OPENAI_TEMPERATURE: '0.5',
+      OPENAI_MAX_TOKENS: '2048',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        omitDefaults: true,
+      },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+
+    expect(body.temperature).toBe(0.5);
+    expect(body.max_output_tokens).toBe(2048);
+  });
+
+  it('should correctly send max_output_tokens: 0 in the request body when explicitly set', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [{ type: 'output_text', text: 'Response' }],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Test that max_output_tokens: 0 is correctly sent (not filtered out by falsy check)
+    // Note: While max_output_tokens: 0 is impractical, it should still be sent if explicitly configured
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        max_output_tokens: 0,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    // max_output_tokens: 0 should be present in the request body
+    expect(body.max_output_tokens).toBe(0);
+    expect('max_output_tokens' in body).toBe(true);
+  });
+
+  it('should strip max_tokens from passthrough for the responses API', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        max_output_tokens: 512,
+        passthrough: { max_tokens: 16000 },
+      },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+
+    expect(body).not.toHaveProperty('max_tokens');
+    expect(body.max_output_tokens).toBe(512);
+  });
+
+  it('should forward prompt caching and include options', async () => {
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        prompt_cache_key: 'shared-prefix',
+        prompt_cache_retention: '24h',
+        include: ['web_search_call.results', 'reasoning.encrypted_content'],
+      },
+    });
+
+    const { body } = await provider.getOpenAiBody('Test prompt');
+    const { body: gpt56Body } = await new OpenAiResponsesProvider('gpt-5.6', {
+      config: { apiKey: 'test-key', prompt_cache_options: { mode: 'explicit', ttl: '30m' } },
+    }).getOpenAiBody('Test prompt');
+
+    expect(body.prompt_cache_key).toBe('shared-prefix');
+    expect(body.prompt_cache_retention).toBe('24h');
+    expect(body.include).toEqual(['web_search_call.results', 'reasoning.encrypted_content']);
+    expect(gpt56Body.prompt_cache_options).toEqual({ mode: 'explicit', ttl: '30m' });
+  });
+
+  it('should handle store parameter correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Stored response',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        store: true,
+      },
+    });
+
+    await provider.callApi('Test prompt');
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"store":true'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+  });
+
+  it('should handle various structured inputs correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Response to structured input',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const structuredInput = JSON.stringify([
+      { role: 'system', content: 'You are a helpful assistant' },
+      { role: 'user', content: 'Hello' },
+    ]);
+
+    await provider.callApi(structuredInput);
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    expect(body.input).toBeDefined();
+
+    const inputStr = JSON.stringify(body.input);
+    expect(inputStr).toContain('You are a helpful assistant');
+    expect(inputStr).toContain('Hello');
+  });
+
+  it('should format JSON schema correctly in request body', async () => {
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        id: 'resp_abc123',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: '{"result": "success"}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const config = {
+      apiKey: 'test-key',
+      response_format: {
+        type: 'json_schema' as const,
+        json_schema: {
+          name: 'TestSchema',
+          strict: true,
+          schema: {
+            type: 'object' as const,
+            properties: {
+              result: { type: 'string' },
+            },
+            required: ['result'],
+            additionalProperties: false,
+          },
+        },
+      },
+    } as any;
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', { config });
+    await provider.callApi('Test prompt');
+
+    expect(vi.mocked(cache.fetchWithCache).mock.calls.length).toBeGreaterThan(0);
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    expect(mockCall).toBeDefined();
+
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    expect(body.text.format.type).toBe('json_schema');
+    expect(body.text.format.name).toBe('TestSchema');
+    expect(body.text.format.schema).toBeDefined();
+    expect(body.text.format.strict).toBe(true);
+  });
+
+  it('should handle JSON object prompt correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'output_text',
+              text: 'Response to object input',
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    const objectInput = JSON.stringify({ query: 'What is the weather?', context: 'San Francisco' });
+    await provider.callApi(objectInput);
+
+    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+    const reqOptions = mockCall[1] as { body: string };
+    const body = JSON.parse(reqOptions.body);
+
+    expect(body.input).toEqual(objectInput);
+  });
+});

test/providers/openai/responses/responseFormat.test.ts

@@ -0,0 +1,378 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider response formats', () => {
+  describe('response format handling', () => {
+    it('should handle json_object format correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: '{"result": "success"}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          response_format: {
+            type: 'json_object',
+          },
+        },
+      });
+
+      await provider.callApi('Test prompt with JSON');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text).toEqual({
+        format: {
+          type: 'json_object',
+        },
+      });
+    });
+
+    it('should handle json_schema format correctly with name parameter', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: '{"result": "success"}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const config = {
+        apiKey: 'test-key',
+        response_format: {
+          type: 'json_schema' as const,
+          json_schema: {
+            name: 'test_schema',
+            strict: true,
+            schema: {
+              type: 'object' as const,
+              properties: {
+                result: { type: 'string' },
+              },
+              required: ['result'],
+              additionalProperties: false,
+            },
+          },
+        },
+      } as any;
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', { config });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text.format.type).toBe('json_schema');
+      expect(body.text.format.name).toBe('test_schema');
+      expect(body.text.format.schema).toBeDefined();
+      expect(body.text.format.strict).toBe(true);
+    });
+
+    it('should handle json_schema format with default name when not provided', async () => {
+      const mockApiResponse = {
+        id: 'resp_def456',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: '{"result": "default name test"}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const config = {
+        apiKey: 'test-key',
+        response_format: {
+          type: 'json_schema' as const,
+          json_schema: {
+            strict: true,
+            schema: {
+              type: 'object' as const,
+              properties: {
+                result: { type: 'string' },
+              },
+              required: ['result'],
+              additionalProperties: false,
+            },
+          },
+        },
+      } as any;
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', { config });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text.format.type).toBe('json_schema');
+      expect(body.text.format.name).toBeTruthy();
+      expect(body.text.format.schema).toBeDefined();
+      expect(body.text.format.strict).toBe(true);
+    });
+
+    it('should handle json_schema format with nested json_schema.schema', async () => {
+      const mockApiResponse = {
+        id: 'resp_ghi789',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: '{"result": "nested schema test"}',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const config = {
+        apiKey: 'test-key',
+        response_format: {
+          type: 'json_schema' as const,
+          json_schema: {
+            name: 'nested_test',
+            strict: true,
+            schema: {
+              type: 'object' as const,
+              properties: {
+                result: { type: 'string' },
+              },
+              required: ['result'],
+              additionalProperties: false,
+            },
+          },
+        } as any,
+      };
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', { config });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text.format.type).toBe('json_schema');
+      expect(body.text.format.name).toBe('nested_test');
+      expect(body.text.format.schema).toBeDefined();
+      expect(body.text.format.strict).toBe(true);
+    });
+
+    it('should handle text format correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Simple text response',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text).toEqual({
+        format: {
+          type: 'text',
+        },
+      });
+    });
+
+    it('should handle external file loading for response_format correctly', async () => {
+      // Test that the provider can be configured with external file syntax
+      // This verifies the type handling for external file references
+      expect(() => {
+        new OpenAiResponsesProvider('gpt-4o', {
+          config: {
+            apiKey: 'test-key',
+            response_format: 'file://./response_format.json' as any,
+          },
+        });
+      }).not.toThrow();
+    });
+
+    it('should handle explicit text format correctly', async () => {
+      const mockApiResponse = {
+        id: 'resp_abc123',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [
+              {
+                type: 'output_text',
+                text: 'Explicit text format response',
+              },
+            ],
+          },
+        ],
+        usage: { input_tokens: 10, output_tokens: 15, total_tokens: 25 },
+      };
+
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: mockApiResponse,
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          response_format: {
+            type: 'text' as any,
+          },
+        },
+      });
+
+      await provider.callApi('Test prompt');
+
+      const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
+      const reqOptions = mockCall[1] as { body: string };
+      const body = JSON.parse(reqOptions.body);
+
+      expect(body.text).toEqual({
+        format: {
+          type: 'text',
+        },
+      });
+    });
+
+    it('should accept external file reference syntax for response_format', async () => {
+      // Test that the provider can be instantiated with external file syntax
+      // without throwing type errors (using type assertion as needed)
+      expect(() => {
+        new OpenAiResponsesProvider('gpt-4o', {
+          config: {
+            apiKey: 'test-key',
+            response_format: 'file://./schema.json' as any,
+          },
+        });
+      }).not.toThrow();
+
+      expect(() => {
+        new OpenAiResponsesProvider('gpt-4o', {
+          config: {
+            apiKey: 'test-key',
+            response_format: 'file://relative/path/schema.json' as any,
+          },
+        });
+      }).not.toThrow();
+
+      expect(() => {
+        new OpenAiResponsesProvider('gpt-4o', {
+          config: {
+            apiKey: 'test-key',
+            response_format: 'file:///absolute/path/schema.json' as any,
+          },
+        });
+      }).not.toThrow();
+    });
+  });
+});

test/providers/openai/responses/setup.ts

@@ -0,0 +1,75 @@
+// Shared harness for the split OpenAI responses provider tests.
+//
+// Each split test file must import this module with a bare side-effect
+// import (`import './setup';`) BEFORE importing the modules under test, so
+// the `vi.mock` calls below register against the worker's module registry
+// before `src/cache` / `src/logger` / `src/python/pythonUtils` load.
+//
+// The top-level `beforeEach` / `afterEach` hooks register into the importing
+// file's test context because `vitest.config.ts` sets `isolate: true`, which
+// re-evaluates this module for every test file. If that flag is ever
+// disabled, the hooks would register only once per worker (in the first
+// file that imports this module) and env/mocks would silently leak across
+// files — update this harness before flipping it.
+import { afterEach, beforeEach, vi } from 'vitest';
+import { mockProcessEnv } from '../../../util/utils';
+
+vi.mock('../../../../src/cache', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    fetchWithCache: vi.fn(),
+  };
+});
+
+vi.mock('../../../../src/logger', () => ({
+  __esModule: true,
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+vi.mock('../../../../src/python/pythonUtils', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    runPython: vi.fn(),
+  };
+});
+
+const ENV_KEYS_TO_CLEAR = [
+  'OPENAI_TEMPERATURE',
+  'OPENAI_MAX_TOKENS',
+  'OPENAI_MAX_COMPLETION_TOKENS',
+  'OPENAI_API_BASE_URL',
+  'OPENAI_BASE_URL',
+  'OPENAI_API_HOST',
+] as const;
+
+type OpenAiEnvKey = (typeof ENV_KEYS_TO_CLEAR)[number];
+
+let restoreOpenAiEnv = () => {};
+
+function resetOpenAiEnv(overrides: Partial<Record<OpenAiEnvKey, string | undefined>> = {}) {
+  restoreOpenAiEnv();
+  restoreOpenAiEnv = mockProcessEnv({
+    ...Object.fromEntries(ENV_KEYS_TO_CLEAR.map((key) => [key, undefined])),
+    ...overrides,
+  });
+}
+
+export function setOpenAiEnv(overrides: Partial<Record<OpenAiEnvKey, string | undefined>>) {
+  resetOpenAiEnv(overrides);
+}
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  resetOpenAiEnv();
+});
+
+afterEach(() => {
+  vi.resetAllMocks();
+  restoreOpenAiEnv();
+  restoreOpenAiEnv = () => {};
+});

test/providers/openai/responses/toolLoading.test.ts

@@ -0,0 +1,107 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider tool loading', () => {
+  describe('tool loading from external files', () => {
+    it('should return loaded tools array in config for downstream validation', async () => {
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'web_search_preview' }],
+        },
+      });
+
+      const context = { prompt: { raw: 'test', label: 'test' }, vars: {} };
+      const { body, config } = await provider.getOpenAiBody('test prompt', context);
+
+      // Verify tools are returned in both body and config
+      expect(body.tools).toEqual([{ type: 'web_search_preview' }]);
+      expect(config.tools).toEqual([{ type: 'web_search_preview' }]);
+      expect(Array.isArray(config.tools)).toBe(true);
+    });
+
+    it('should return undefined tools when not configured', async () => {
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+        },
+      });
+
+      const context = { prompt: { raw: 'test', label: 'test' }, vars: {} };
+      const { body, config } = await provider.getOpenAiBody('test prompt', context);
+
+      expect(body.tools).toBeUndefined();
+      expect(config.tools).toBeUndefined();
+    });
+
+    it('should throw clear error for Python tool files', async () => {
+      const provider = new OpenAiResponsesProvider('gpt-4o', {
+        config: {
+          apiKey: 'test-key',
+          tools: 'file://tools.py:get_tools' as any,
+        },
+      });
+
+      const context = { prompt: { raw: 'test', label: 'test' }, vars: {} };
+      await expect(provider.getOpenAiBody('test prompt', context)).rejects.toThrow(
+        /Failed to load tools/,
+      );
+    });
+
+    it('should allow deep-research validation to work with loaded tools', async () => {
+      const provider = new OpenAiResponsesProvider('o4-mini-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [{ type: 'web_search_preview' }],
+        },
+      });
+
+      // Mock the API call
+      vi.mocked(cache.fetchWithCache).mockResolvedValue({
+        data: {
+          id: 'resp_123',
+          object: 'response',
+          status: 'completed',
+          model: 'o4-mini-deep-research',
+          output: [
+            {
+              type: 'message',
+              content: [{ type: 'output_text', text: 'Response' }],
+            },
+          ],
+          usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      // This should not throw TypeError because config.tools is now an array
+      const result = await provider.callApi('test');
+      expect(result.error).toBeUndefined();
+    });
+
+    it('should return error for deep-research without web_search_preview', async () => {
+      const provider = new OpenAiResponsesProvider('o4-mini-deep-research', {
+        config: {
+          apiKey: 'test-key',
+          tools: [
+            {
+              type: 'function',
+              function: { name: 'test', parameters: { type: 'object', properties: {} } },
+            },
+          ],
+        },
+      });
+
+      const result = await provider.callApi('test');
+
+      expect(result.error).toContain('requires the web_search_preview tool');
+    });
+  });
+});

test/providers/openai/responses/tools.test.ts

@@ -0,0 +1,441 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+describe('OpenAiResponsesProvider tool handling', () => {
+  it('should handle tool calling correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'tool_call',
+              name: 'get_weather',
+              id: 'call_123',
+              input: { location: 'San Francisco' },
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const tools = [
+      {
+        type: 'function' as const,
+        function: {
+          name: 'get_weather',
+          description: 'Get the current weather in a given location',
+          parameters: {
+            type: 'object' as const,
+            properties: {
+              location: { type: 'string' },
+            },
+            required: ['location'],
+          },
+        },
+      },
+    ];
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        tools,
+      },
+    });
+
+    const result = await provider.callApi("What's the weather in San Francisco?");
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"tools":[{'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+
+    expect(result.raw).toHaveProperty('output');
+    expect(JSON.stringify(result.raw)).toContain('get_weather');
+    expect(JSON.stringify(result.raw)).toContain('San Francisco');
+  });
+
+  it('should handle parallel tool calls correctly', async () => {
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'tool_call',
+              name: 'get_weather',
+              id: 'call_123',
+              input: { location: 'San Francisco' },
+            },
+          ],
+        },
+      ],
+      parallel_tool_calls: true,
+      usage: { input_tokens: 20, output_tokens: 15, total_tokens: 35 },
+    };
+
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        tools: [
+          {
+            type: 'function' as const,
+            function: {
+              name: 'get_weather',
+              description: 'Get weather',
+              parameters: {
+                type: 'object' as const,
+                properties: {
+                  location: { type: 'string' },
+                },
+              },
+            },
+          },
+        ],
+        parallel_tool_calls: true,
+      },
+    });
+
+    await provider.callApi('Weather?');
+
+    expect(cache.fetchWithCache).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        body: expect.stringContaining('"parallel_tool_calls":true'),
+      }),
+      expect.any(Number),
+      'json',
+      undefined,
+      undefined,
+    );
+  });
+
+  it('should handle top-level function calls correctly', async () => {
+    // Mock API response with a top-level function call
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'function_call',
+          name: 'get_weather',
+          id: 'call_123',
+          input: { location: 'San Francisco' },
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify the top-level function call is returned as a stringified JSON
+    expect(result.output).toContain('get_weather');
+    expect(result.output).toContain('San Francisco');
+    // It should be a stringified object
+    expect(() => JSON.parse(result.output)).not.toThrow();
+    const parsed = JSON.parse(result.output);
+    expect(parsed.type).toBe('function_call');
+  });
+
+  // Test for tool calls in assistant messages (lines 205-207)
+
+  it('should handle tool calls in assistant messages correctly', async () => {
+    // Mock API response with a tool_use in assistant message
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'message',
+          role: 'assistant',
+          content: [
+            {
+              type: 'tool_use',
+              name: 'get_weather',
+              id: 'call_123',
+              input: { location: 'New York' },
+            },
+          ],
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify the tool call is returned as a stringified JSON
+    expect(result.output).toContain('tool_use');
+    expect(result.output).toContain('New York');
+    // It should be a stringified object
+    expect(() => JSON.parse(result.output)).not.toThrow();
+    const parsed = JSON.parse(result.output);
+    expect(parsed.type).toBe('tool_use');
+  });
+
+  // Test for tool results (lines 210-212)
+
+  it('should handle tool results correctly', async () => {
+    // Mock API response with a tool result
+    const mockApiResponse = {
+      id: 'resp_abc123',
+      status: 'completed',
+      model: 'gpt-4o',
+      output: [
+        {
+          type: 'tool_result',
+          id: 'result_123',
+          tool_call_id: 'call_123',
+          output: { temperature: 72, conditions: 'Sunny' },
+        },
+      ],
+      usage: { input_tokens: 15, output_tokens: 10, total_tokens: 25 },
+    };
+
+    // Setup mock for fetchWithCache
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    // Initialize the provider
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+      },
+    });
+
+    // Call the API
+    const result = await provider.callApi('Test prompt');
+
+    // Verify the tool result is returned as a stringified JSON
+    expect(result.output).toContain('tool_result');
+    expect(result.output).toContain('Sunny');
+    // It should be a stringified object
+    expect(() => JSON.parse(result.output)).not.toThrow();
+    const parsed = JSON.parse(result.output);
+    expect(parsed.type).toBe('tool_result');
+  });
+
+  it('should include observable web and file search fees in responses cost', async () => {
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        id: 'resp_cost123',
+        status: 'completed',
+        model: 'gpt-5-mini',
+        output: [
+          {
+            type: 'web_search_call',
+            action: { type: 'search', query: 'pricing' },
+            status: 'completed',
+          },
+          {
+            type: 'file_search_call',
+            status: 'completed',
+          },
+        ],
+        usage: {
+          input_tokens: 1_000,
+          output_tokens: 100,
+          total_tokens: 1_100,
+        },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-5-mini', {
+      config: {
+        apiKey: 'test-key',
+        tools: [{ type: 'web_search_preview' }],
+      },
+    });
+
+    const result = await provider.callApi('Search my files');
+
+    expect(result.cost).toBeCloseTo((1_000 * 0.25 + 100 * 2) / 1e6 + 0.0125, 10);
+  });
+
+  it('should price observable web search fees from effective passthrough tools', async () => {
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        id: 'resp_cost456',
+        status: 'completed',
+        model: 'gpt-4o',
+        output: [
+          {
+            type: 'web_search_call',
+            action: { type: 'search', query: 'pricing' },
+            status: 'completed',
+          },
+        ],
+        usage: {
+          input_tokens: 0,
+          output_tokens: 0,
+          total_tokens: 0,
+        },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: {
+        apiKey: 'test-key',
+        tools: [{ type: 'web_search_preview' }],
+        passthrough: { tools: [{ type: 'web_search' }] },
+      },
+    });
+
+    const result = await provider.callApi('Search the web');
+
+    expect(result.cost).toBeCloseTo(0.01, 10);
+  });
+
+  it('should price from the tier returned by OpenAI rather than the requested tier', async () => {
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        id: 'resp_priority123',
+        status: 'completed',
+        model: 'gpt-5-mini',
+        service_tier: 'priority',
+        output: [
+          {
+            type: 'message',
+            role: 'assistant',
+            content: [{ type: 'output_text', text: 'Done' }],
+          },
+        ],
+        usage: {
+          input_tokens: 1_000,
+          output_tokens: 100,
+          total_tokens: 1_100,
+        },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-5-mini', {
+      config: {
+        apiKey: 'test-key',
+        service_tier: 'flex',
+      },
+    });
+
+    const result = await provider.callApi('Do the thing');
+
+    expect(result.cost).toBeCloseTo((1_000 * 0.45 + 100 * 3.6) / 1e6, 10);
+  });
+
+  it('should not invent code-interpreter session fees from response output alone', async () => {
+    const mockApiResponse = {
+      id: 'resp_ci123',
+      status: 'completed',
+      model: 'gpt-5-mini',
+      output: [
+        {
+          type: 'code_interpreter_call',
+          container_id: 'cntr_a',
+          status: 'completed',
+        },
+      ],
+      usage: {
+        input_tokens: 1_000,
+        output_tokens: 100,
+        total_tokens: 1_100,
+      },
+    };
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: mockApiResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-5-mini', {
+      config: {
+        apiKey: 'test-key',
+        tools: [
+          { type: 'code_interpreter', container: { type: 'auto', memory_limit: '4g' } } as any,
+        ],
+      },
+    });
+
+    const result = await provider.callApi('Use python');
+
+    expect(result.cost).toBeCloseTo((1_000 * 0.25 + 100 * 2) / 1e6, 10);
+  });
+});

test/providers/openai/responses/tracing.test.ts

@@ -0,0 +1,166 @@
+// Load-bearing: registers shared vi.mock / beforeEach hooks before any
+// module-under-test import below. See ./setup.ts for details.
+import './setup';
+
+import { trace } from '@opentelemetry/api';
+import { describe, expect, it, vi } from 'vitest';
+import * as cache from '../../../../src/cache';
+import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
+
+interface RecordedSpan {
+  name: string;
+  attributes: Record<string, any>;
+  status?: { code: number; message?: string };
+  ended: boolean;
+}
+
+function installTracerSpy(): RecordedSpan[] {
+  const spans: RecordedSpan[] = [];
+  const make = (name: string, attributes: Record<string, any> = {}) => {
+    const entry: RecordedSpan = { name, attributes: { ...attributes }, ended: false };
+    spans.push(entry);
+    return {
+      setAttribute: (key: string, value: unknown) => {
+        entry.attributes[key] = value;
+      },
+      setAttributes: (attrs: Record<string, unknown>) => Object.assign(entry.attributes, attrs),
+      setStatus: (status: { code: number; message?: string }) => {
+        entry.status = status;
+      },
+      end: () => {
+        entry.ended = true;
+      },
+      recordException: () => undefined,
+      addEvent: () => undefined,
+      spanContext: () => ({ traceId: 'x', spanId: 'y' }),
+      isRecording: () => true,
+      updateName: () => undefined,
+    };
+  };
+  vi.spyOn(trace, 'getTracer').mockReturnValue({
+    startSpan: (name: string, options?: { attributes?: Record<string, unknown> }) =>
+      make(name, options?.attributes),
+    startActiveSpan: (...args: any[]) => {
+      const name = args[0];
+      const options = typeof args[1] === 'object' ? args[1] : undefined;
+      const callback = args[args.length - 1];
+      return callback(make(name, options?.attributes));
+    },
+  } as any);
+  return spans;
+}
+
+const successResponse = {
+  id: 'resp_abc123',
+  object: 'response',
+  created_at: 1234567890,
+  status: 'completed',
+  model: 'gpt-4o',
+  output: [
+    {
+      type: 'message',
+      id: 'msg_abc123',
+      status: 'completed',
+      role: 'assistant',
+      content: [{ type: 'output_text', text: 'This is a test response' }],
+    },
+  ],
+  usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
+};
+
+describe('OpenAiResponsesProvider tracing', () => {
+  it('emits a chat <model> span with request and response attributes', async () => {
+    const spans = installTracerSpy();
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: successResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', { config: { apiKey: 'test-key' } });
+    const result = await provider.callApi('Test prompt');
+    expect(result.output).toBe('This is a test response');
+
+    const chatSpan = spans.find((span) => span.name === 'chat gpt-4o');
+    expect(chatSpan).toBeDefined();
+    expect(chatSpan?.attributes).toMatchObject({
+      'gen_ai.system': 'openai',
+      'gen_ai.operation.name': 'chat',
+      'gen_ai.request.model': 'gpt-4o',
+      'promptfoo.provider.id': 'openai:gpt-4o',
+    });
+    expect(chatSpan?.attributes['gen_ai.usage.total_tokens']).toBe(30);
+    expect(chatSpan?.ended).toBe(true);
+    // SpanStatusCode.OK === 1
+    expect(chatSpan?.status?.code).toBe(1);
+  });
+
+  it('emits reasoning token usage (completionDetails) on the chat span', async () => {
+    const spans = installTracerSpy();
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: {
+        ...successResponse,
+        usage: {
+          total_tokens: 50,
+          prompt_tokens: 10,
+          completion_tokens: 40,
+          completion_tokens_details: { reasoning_tokens: 32 },
+        },
+      },
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('o3', { config: { apiKey: 'test-key' } });
+    await provider.callApi('Test prompt');
+
+    const chatSpan = spans.find((span) => span.name === 'chat o3');
+    expect(chatSpan).toBeDefined();
+    // The reasoning-token detail must survive extractProviderResponseAttributes
+    // and reach the span (it previously dropped tokenUsage.completionDetails).
+    expect(chatSpan?.attributes['gen_ai.usage.reasoning_tokens']).toBe(32);
+    expect(chatSpan?.attributes['gen_ai.usage.output_tokens']).toBe(40);
+  });
+
+  it('records request params from the resolved request body', async () => {
+    const spans = installTracerSpy();
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: successResponse,
+      cached: false,
+      status: 200,
+      statusText: 'OK',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', {
+      config: { apiKey: 'test-key', temperature: 0.3, top_p: 0.9, max_output_tokens: 256 },
+    });
+    await provider.callApi('Test prompt');
+
+    const chatSpan = spans.find((span) => span.name === 'chat gpt-4o');
+    expect(chatSpan?.attributes['gen_ai.request.temperature']).toBe(0.3);
+    expect(chatSpan?.attributes['gen_ai.request.top_p']).toBe(0.9);
+    expect(chatSpan?.attributes['gen_ai.request.max_tokens']).toBe(256);
+  });
+
+  it('marks the chat span ERROR when the API returns an error', async () => {
+    const spans = installTracerSpy();
+    vi.mocked(cache.fetchWithCache).mockResolvedValue({
+      data: { error: { message: 'rate limited', type: 'rate_limit_error' } },
+      cached: false,
+      status: 429,
+      statusText: 'Too Many Requests',
+    });
+
+    const provider = new OpenAiResponsesProvider('gpt-4o', { config: { apiKey: 'test-key' } });
+    const result = await provider.callApi('Test prompt');
+    expect(result.error).toBeDefined();
+
+    const chatSpan = spans.find((span) => span.name === 'chat gpt-4o');
+    expect(chatSpan).toBeDefined();
+    // SpanStatusCode.ERROR === 2
+    expect(chatSpan?.status?.code).toBe(2);
+    expect(chatSpan?.ended).toBe(true);
+  });
+});

test/providers/openai/shared.ts

@@ -0,0 +1,12 @@
+export function getOpenAiMissingApiKeyMessage(envVar = 'OPENAI_API_KEY'): string {
+  return `API key is not set. Set the ${envVar} environment variable or add \`apiKey\` to the provider config.`;
+}
+
+export function restoreEnvVar(name: string, value: string | undefined): void {
+  if (value === undefined) {
+    Reflect.deleteProperty(process.env, name);
+    return;
+  }
+
+  Object.assign(process.env, { [name]: value });
+}

test/providers/openai/transcription.test.ts

@@ -0,0 +1,755 @@
+import fs from 'fs';
+
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { fetchWithCache } from '../../../src/cache';
+import { OpenAiTranscriptionProvider } from '../../../src/providers/openai/transcription';
+import { mockGlobal, mockProcessEnv } from '../../util/utils';
+import { getOpenAiMissingApiKeyMessage } from './shared';
+
+vi.mock('../../../src/cache', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    fetchWithCache: vi.fn(),
+  };
+});
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+const fsMocks = vi.hoisted(() => ({
+  existsSync: vi.fn(),
+  readFileSync: vi.fn(),
+}));
+
+vi.mock('fs', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('fs')>();
+  return {
+    ...actual,
+    default: {
+      ...actual,
+      ...fsMocks,
+    },
+    ...fsMocks,
+  };
+});
+vi.mock('fs/promises', () => {
+  // Async wrapper around the sync mock so the returned value is a real Promise,
+  // matching the actual fs/promises.readFile API.
+  const readFile = vi.fn(async (filePath: any, encoding?: any) =>
+    encoding === undefined
+      ? fsMocks.readFileSync(filePath)
+      : fsMocks.readFileSync(filePath, encoding),
+  );
+  return {
+    default: {
+      readFile,
+    },
+    readFile,
+  };
+});
+
+class MockFile {
+  constructor(
+    public parts: any[],
+    public name: string,
+    public options?: FilePropertyBag,
+  ) {}
+}
+
+class MockFormData {
+  private data: Map<string, any> = new Map();
+
+  append(key: string, value: any) {
+    this.data.set(key, value);
+  }
+
+  get(key: string) {
+    return this.data.get(key);
+  }
+
+  has(key: string) {
+    return this.data.has(key);
+  }
+}
+
+const restoreFile = mockGlobal('File', MockFile as unknown as typeof File);
+const restoreFormData = mockGlobal('FormData', MockFormData as unknown as typeof FormData);
+
+afterAll(() => {
+  restoreFormData();
+  restoreFile();
+});
+
+describe('OpenAiTranscriptionProvider', () => {
+  const mockTranscriptionResponse = {
+    data: {
+      task: 'transcribe',
+      text: 'This is a test transcription.',
+      duration: 120, // 2 minutes
+      language: 'en',
+      segments: [
+        {
+          id: 0,
+          start: 0,
+          end: 60,
+          text: 'This is a test',
+          avg_logprob: -0.3,
+          compression_ratio: 1.2,
+          no_speech_prob: 0.01,
+        },
+        {
+          id: 1,
+          start: 60,
+          end: 120,
+          text: 'transcription.',
+          avg_logprob: -0.4,
+          compression_ratio: 1.1,
+          no_speech_prob: 0.02,
+        },
+      ],
+    },
+    cached: false,
+    status: 200,
+    statusText: 'OK',
+  };
+
+  const mockDiarizedResponse = {
+    data: {
+      task: 'transcribe',
+      duration: 180, // 3 minutes
+      language: 'en',
+      segments: [
+        {
+          speaker: 'Speaker 1',
+          text: 'Hello, how are you?',
+          start: 0.0,
+          end: 2.5,
+          avg_logprob: -0.25,
+          compression_ratio: 1.3,
+          no_speech_prob: 0.005,
+        },
+        {
+          speaker: 'Speaker 2',
+          text: "I'm doing great, thanks!",
+          start: 2.5,
+          end: 5.0,
+          avg_logprob: -0.35,
+          compression_ratio: 1.25,
+          no_speech_prob: 0.01,
+        },
+      ],
+      speakers: ['Speaker 1', 'Speaker 2'],
+    },
+    cached: false,
+    status: 200,
+    statusText: 'OK',
+  };
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return Buffer.from('mock audio data');
+    });
+    vi.mocked(fetchWithCache).mockResolvedValue(mockTranscriptionResponse);
+  });
+
+  describe('Basic functionality', () => {
+    it('should transcribe audio successfully', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(fs.readFileSync).toHaveBeenCalledWith('/path/to/audio.mp3');
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.stringContaining('/audio/transcriptions'),
+        expect.objectContaining({
+          method: 'POST',
+          headers: expect.objectContaining({
+            Authorization: 'Bearer test-key',
+            'X-OpenAI-Originator': 'promptfoo',
+          }),
+        }),
+        expect.any(Number),
+        'json',
+        undefined,
+      );
+
+      expect(result).toEqual({
+        output: 'This is a test transcription.',
+        cached: false,
+        cost: 0.012, // 2 minutes * $0.006/min
+        metadata: {
+          task: 'transcribe',
+          duration: 120,
+          language: 'en',
+          segments: 2,
+          avgLogprob: -0.35, // Average of -0.3 and -0.4
+          avgCompressionRatio: 1.15, // Average of 1.2 and 1.1
+          avgNoSpeechProb: 0.015, // Average of 0.01 and 0.02
+        },
+      });
+    });
+
+    it('should use cached response', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        ...mockTranscriptionResponse,
+        cached: true,
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toEqual({
+        output: 'This is a test transcription.',
+        cached: true,
+        cost: 0, // Cost is 0 for cached responses
+        metadata: {
+          task: 'transcribe',
+          duration: 120,
+          language: 'en',
+          segments: 2,
+          avgLogprob: -0.35,
+          avgCompressionRatio: 1.15,
+          avgNoSpeechProb: 0.015,
+        },
+      });
+    });
+
+    it('should calculate cost correctly for gpt-4o-mini-transcribe', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-mini-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.cost).toBe(0.006); // 2 minutes * $0.003/min
+    });
+
+    it('should calculate cost correctly for gpt-4o-mini-transcribe-2025-12-15', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-mini-transcribe-2025-12-15', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.cost).toBe(0.006); // 2 minutes * $0.003/min
+    });
+
+    it('should calculate cost correctly for whisper-1', async () => {
+      const provider = new OpenAiTranscriptionProvider('whisper-1', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.cost).toBe(0.012); // 2 minutes * $0.006/min
+    });
+
+    it('should correctly use ID passed during construction', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+        id: 'custom-provider-id',
+      });
+
+      expect(provider.id()).toBe('custom-provider-id');
+    });
+
+    it('should generate correct default ID', () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(provider.id()).toBe('openai:transcription:gpt-4o-transcribe');
+    });
+
+    it('should generate correct default ID for dated diarization snapshots', () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe-diarize-2025-10-15', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(provider.id()).toBe('openai:transcription:gpt-4o-transcribe-diarize-2025-10-15');
+    });
+
+    it('should throw an error if API key is not set', async () => {
+      const restoreEnv = mockProcessEnv({ OPENAI_API_KEY: undefined });
+
+      try {
+        const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe');
+
+        await expect(provider.callApi('/path/to/audio.mp3')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage(),
+        );
+      } finally {
+        restoreEnv();
+      }
+    });
+
+    it('should use custom apiKeyEnvar in missing API key errors', async () => {
+      const restoreEnv = mockProcessEnv({
+        OPENAI_API_KEY: undefined,
+        CUSTOM_TRANSCRIPTION_API_KEY: undefined,
+      });
+
+      try {
+        const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+          config: {
+            apiKeyEnvar: 'CUSTOM_TRANSCRIPTION_API_KEY',
+          },
+          env: {
+            OPENAI_API_KEY: undefined,
+            CUSTOM_TRANSCRIPTION_API_KEY: undefined,
+          },
+        });
+
+        await expect(provider.callApi('/path/to/audio.mp3')).rejects.toThrow(
+          getOpenAiMissingApiKeyMessage('CUSTOM_TRANSCRIPTION_API_KEY'),
+        );
+      } finally {
+        restoreEnv();
+      }
+    });
+  });
+
+  describe('Diarization support', () => {
+    it('should handle diarized transcription', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe-diarize', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue(mockDiarizedResponse);
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.output).toBe(
+        "[0.00s - 2.50s] Speaker 1: Hello, how are you?\n[2.50s - 5.00s] Speaker 2: I'm doing great, thanks!",
+      );
+      expect(result.cached).toBe(false);
+      expect(result.cost).toBeCloseTo(0.018, 5); // 3 minutes * $0.006/min
+      expect(result.metadata).toEqual({
+        task: 'transcribe',
+        duration: 180,
+        language: 'en',
+        segments: 2,
+        avgLogprob: -0.3, // Average of -0.25 and -0.35
+        avgCompressionRatio: 1.275, // Average of 1.3 and 1.25
+        avgNoSpeechProb: 0.0075, // Average of 0.005 and 0.01
+        speakers: ['Speaker 1', 'Speaker 2'],
+      });
+    });
+
+    it('should include num_speakers option for diarization', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe-diarize', {
+        config: {
+          apiKey: 'test-key',
+          num_speakers: 2,
+        },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue(mockDiarizedResponse);
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      // Since we're using FormData, we can't easily inspect the body
+      // Just verify the call was made
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should include speaker_labels option for diarization', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe-diarize', {
+        config: {
+          apiKey: 'test-key',
+          speaker_labels: ['Alice', 'Bob'],
+        },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue(mockDiarizedResponse);
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+  });
+
+  describe('Error handling', () => {
+    it('should handle missing audio file', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fs.readFileSync).mockImplementation(function () {
+        throw Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' });
+      });
+
+      const result = await provider.callApi('/path/to/missing.mp3');
+
+      expect(result).toEqual({
+        error: 'Audio file not found: /path/to/missing.mp3',
+      });
+    });
+
+    it('should handle API errors', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const errorResponse = {
+        data: { error: 'Invalid audio format' },
+        cached: false,
+        status: 400,
+        statusText: 'Bad Request',
+      };
+
+      vi.mocked(fetchWithCache).mockResolvedValue(errorResponse);
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('Invalid audio format');
+    });
+
+    it('should handle HTTP errors', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: 'Error message',
+        cached: false,
+        status: 500,
+        statusText: 'Internal Server Error',
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('API error: 500 Internal Server Error');
+    });
+
+    it('should handle fetch errors', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockRejectedValue(new Error('Network error'));
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('API call error: Error: Network error');
+    });
+
+    it('should handle missing transcription in response', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: { duration: 120 },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('No transcription returned from API');
+    });
+
+    it('should handle transcription error in catch block', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockImplementation(function () {
+        throw new Error('Unexpected error');
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result).toHaveProperty('error');
+      expect(result.error).toContain('API call error: Error: Unexpected error');
+    });
+  });
+
+  describe('Configuration options', () => {
+    it('should include language option', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          language: 'es',
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should include prompt option', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          prompt: 'This is a technical discussion about AI.',
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should include temperature option', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          temperature: 0.5,
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should include timestamp_granularities option', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          timestamp_granularities: ['word', 'segment'],
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should include organization ID in headers when provided', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          organization: 'test-org',
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            'OpenAI-Organization': 'test-org',
+          }),
+        }),
+        expect.any(Number),
+        'json',
+        undefined,
+      );
+    });
+
+    it('should merge prompt config with provider config', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key', temperature: 0.5 },
+      });
+
+      const context = {
+        prompt: {
+          raw: '/path/to/audio.mp3',
+          config: { temperature: 0.8 },
+          label: 'test',
+        },
+        vars: {},
+      };
+
+      await provider.callApi('/path/to/audio.mp3', context);
+
+      // Config should be merged with prompt config taking precedence
+      expect(fetchWithCache).toHaveBeenCalled();
+    });
+
+    it('should use custom API URL when provided', async () => {
+      const customApiUrl = 'https://custom-openai.example.com/v1';
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: {
+          apiKey: 'test-key',
+          apiBaseUrl: customApiUrl,
+        },
+      });
+
+      await provider.callApi('/path/to/audio.mp3');
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        `${customApiUrl}/audio/transcriptions`,
+        expect.any(Object),
+        expect.any(Number),
+        'json',
+        undefined,
+      );
+    });
+
+    it('should handle bustCache from context', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const context = {
+        bustCache: true,
+        prompt: { raw: '/path/to/audio.mp3', label: 'test' },
+        vars: {},
+      };
+
+      await provider.callApi('/path/to/audio.mp3', context);
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        expect.any(Number),
+        'json',
+        true,
+      );
+    });
+
+    it('should handle debug mode from context', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      const context = {
+        debug: true,
+        prompt: { raw: '/path/to/audio.mp3', label: 'test' },
+        vars: {},
+      };
+
+      await provider.callApi('/path/to/audio.mp3', context);
+
+      expect(fetchWithCache).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object),
+        expect.any(Number),
+        'json',
+        true,
+      );
+    });
+  });
+
+  describe('Model validation', () => {
+    it('should accept known transcription models', () => {
+      const models = [
+        'gpt-4o-transcribe',
+        'gpt-4o-mini-transcribe',
+        'gpt-4o-mini-transcribe-2025-12-15',
+        'gpt-4o-transcribe-diarize',
+        'gpt-4o-transcribe-diarize-2025-10-15',
+        'whisper-1',
+      ];
+
+      models.forEach((model) => {
+        const provider = new OpenAiTranscriptionProvider(model, {
+          config: { apiKey: 'test-key' },
+        });
+        expect(provider.id()).toBe(`openai:transcription:${model}`);
+      });
+    });
+
+    it('should allow unknown transcription models with debug log', () => {
+      const provider = new OpenAiTranscriptionProvider('unknown-model', {
+        config: { apiKey: 'test-key' },
+      });
+
+      expect(provider.id()).toBe('openai:transcription:unknown-model');
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle zero duration audio', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: {
+          text: 'Test',
+          duration: 0,
+          language: 'en',
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.cost).toBe(0);
+    });
+
+    it('should leave cost undefined when the API omits duration', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: {
+          text: 'Test',
+          language: 'en',
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.cost).toBeUndefined();
+      expect(result.metadata?.duration).toBeUndefined();
+    });
+
+    it('should handle diarized segments with missing fields', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe-diarize', {
+        config: { apiKey: 'test-key' },
+      });
+
+      vi.mocked(fetchWithCache).mockResolvedValue({
+        data: {
+          duration: 60,
+          language: 'en',
+          segments: [
+            {
+              // Missing speaker, start, end fields
+              text: 'Test text',
+            },
+          ],
+        },
+        cached: false,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await provider.callApi('/path/to/audio.mp3');
+
+      expect(result.output).toBe('[0.00s - 0.00s] Unknown: Test text');
+    });
+
+    it('should trim whitespace from audio file path', async () => {
+      const provider = new OpenAiTranscriptionProvider('gpt-4o-transcribe', {
+        config: { apiKey: 'test-key' },
+      });
+
+      await provider.callApi('  /path/to/audio.mp3  ');
+
+      expect(fs.readFileSync).toHaveBeenCalledWith('/path/to/audio.mp3');
+    });
+  });
+});

test/providers/openai/util.test.ts

@@ -0,0 +1,877 @@
+import OpenAI from 'openai';
+import { describe, expect, it, vi } from 'vitest';
+import {
+  calculateOpenAICost,
+  failApiCall,
+  formatOpenAiError,
+  getTokenUsage,
+  OPENAI_CHAT_MODELS,
+  OPENAI_REALTIME_MODELS,
+  OPENAI_RESPONSES_ONLY_MODELS,
+  validateFunctionCall,
+} from '../../../src/providers/openai/util';
+
+vi.mock('../../../src/cache');
+
+describe('failApiCall', () => {
+  it('should format OpenAI API errors', () => {
+    const error = new OpenAI.APIError(400, {}, 'Bad request', new Headers());
+    Object.defineProperty(error, 'type', {
+      value: 'invalid_request_error',
+      writable: true,
+      configurable: true,
+    });
+    Object.defineProperty(error, 'message', {
+      value: 'Bad request',
+      writable: true,
+      configurable: true,
+    });
+    Object.defineProperty(error, 'status', {
+      value: 400,
+      writable: true,
+      configurable: true,
+    });
+
+    const result = failApiCall(error);
+    expect(result).toEqual({
+      error: `API error: invalid_request_error 400 Bad request`,
+    });
+  });
+
+  it('should format generic errors', () => {
+    const error = new Error('Network error');
+    const result = failApiCall(error);
+    expect(result).toEqual({
+      error: `API error: Error: Network error`,
+    });
+  });
+});
+
+describe('getTokenUsage', () => {
+  it('should return token usage for non-cached response', () => {
+    const data = {
+      usage: {
+        total_tokens: 100,
+        prompt_tokens: 40,
+        completion_tokens: 60,
+      },
+    };
+
+    const result = getTokenUsage(data, false);
+    expect(result).toEqual({
+      total: 100,
+      prompt: 40,
+      completion: 60,
+      numRequests: 1,
+    });
+  });
+
+  it('should return cached token usage', () => {
+    const data = {
+      usage: {
+        total_tokens: 100,
+      },
+    };
+
+    const result = getTokenUsage(data, true);
+    // Cached responses don't count as a new request
+    expect(result).toEqual({
+      cached: 100,
+      total: 100,
+    });
+  });
+
+  it('should handle missing usage data', () => {
+    const data = {};
+    const result = getTokenUsage(data, false);
+    expect(result).toEqual({});
+  });
+
+  it('should handle completion tokens details', () => {
+    const data = {
+      usage: {
+        total_tokens: 100,
+        prompt_tokens: 40,
+        completion_tokens: 60,
+        completion_tokens_details: {
+          reasoning_tokens: 20,
+          accepted_prediction_tokens: 30,
+          rejected_prediction_tokens: 10,
+        },
+      },
+    };
+
+    const result = getTokenUsage(data, false);
+    expect(result).toEqual({
+      total: 100,
+      prompt: 40,
+      completion: 60,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 20,
+        acceptedPrediction: 30,
+        rejectedPrediction: 10,
+      },
+    });
+  });
+
+  it('should preserve provider-side cache read and write tokens', () => {
+    const data = {
+      usage: {
+        total_tokens: 100,
+        prompt_tokens: 40,
+        completion_tokens: 60,
+        prompt_tokens_details: {
+          cached_tokens: 32,
+          cache_write_tokens: 4,
+        },
+      },
+    };
+
+    const result = getTokenUsage(data, false);
+    expect(result).toEqual({
+      total: 100,
+      prompt: 40,
+      completion: 60,
+      numRequests: 1,
+      completionDetails: {
+        cacheReadInputTokens: 32,
+        cacheCreationInputTokens: 4,
+      },
+    });
+  });
+
+  it.each([
+    ['top-level positive', { cache_write_input_tokens: 4 }, 4],
+    ['nested explicit zero', { prompt_tokens_details: { cache_write_tokens: 0 } }, 0],
+  ])('should preserve %s cache-write usage', (_name, usageDetails, expected) => {
+    const result = getTokenUsage(
+      {
+        usage: {
+          total_tokens: 100,
+          prompt_tokens: 40,
+          completion_tokens: 60,
+          ...usageDetails,
+        },
+      },
+      false,
+    );
+
+    expect(result.completionDetails).toEqual({
+      cacheCreationInputTokens: expected,
+    });
+  });
+});
+
+describe('calculateOpenAICost', () => {
+  it('should calculate cost correctly for TTS model gpt-4o-mini-tts', () => {
+    const cost = calculateOpenAICost('gpt-4o-mini-tts', {}, 1000, 0, 0, 500);
+    expect(cost).toBeCloseTo((1000 * 0.6 + 500 * 12) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for TTS model gpt-4o-mini-tts-2025-12-15', () => {
+    const cost = calculateOpenAICost('gpt-4o-mini-tts-2025-12-15', {}, 1000, 0, 0, 500);
+    expect(cost).toBeCloseTo((1000 * 0.6 + 500 * 12) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for search preview model gpt-4o-search-preview', () => {
+    const cost = calculateOpenAICost('gpt-4o-search-preview', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for search preview model gpt-4o-search-preview-2025-03-11', () => {
+    const cost = calculateOpenAICost('gpt-4o-search-preview-2025-03-11', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for mini search preview model gpt-4o-mini-search-preview', () => {
+    const cost = calculateOpenAICost('gpt-4o-mini-search-preview', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.15 + 500 * 0.6) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for computer use model computer-use-preview', () => {
+    const cost = calculateOpenAICost('computer-use-preview', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 3 + 500 * 12) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4-1106-vision-preview', () => {
+    const cost = calculateOpenAICost('gpt-4-1106-vision-preview', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 10 + 500 * 30) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4o-realtime-preview-2024-10-01', () => {
+    const cost = calculateOpenAICost('gpt-4o-realtime-preview-2024-10-01', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 5 + 500 * 20) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4o-realtime-preview-2024-12-17', () => {
+    const cost = calculateOpenAICost('gpt-4o-realtime-preview-2024-12-17', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 5 + 500 * 20) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4o-mini-realtime-preview-2024-12-17', () => {
+    const cost = calculateOpenAICost('gpt-4o-mini-realtime-preview-2024-12-17', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.6 + 500 * 2.4) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-realtime', () => {
+    const cost = calculateOpenAICost('gpt-realtime', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 4 + 500 * 16) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-realtime-1.5', () => {
+    const cost = calculateOpenAICost('gpt-realtime-1.5', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 4 + 500 * 16) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-realtime-2025-08-28', () => {
+    const cost = calculateOpenAICost('gpt-realtime-2025-08-28', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 4 + 500 * 16) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-realtime-2', () => {
+    const cost = calculateOpenAICost('gpt-realtime-2', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 4 + 500 * 24) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly with audio tokens', () => {
+    const cost = calculateOpenAICost('gpt-4o-audio-preview', {}, 1000, 500, 200, 100);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 10 + 200 * 40 + 100 * 80) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly with audio tokens for gpt-audio-1.5', () => {
+    const cost = calculateOpenAICost('gpt-audio-1.5', {}, 1000, 500, 200, 100);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 10 + 200 * 32 + 100 * 64) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly with audio tokens for gpt-audio (repriced to $32/$64)', () => {
+    const cost = calculateOpenAICost('gpt-audio', {}, 1000, 500, 200, 100);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 10 + 200 * 32 + 100 * 64) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for the chat-latest alias', () => {
+    const cost = calculateOpenAICost('chat-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 5 + 500 * 30) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-audio-mini-2025-12-15', () => {
+    const cost = calculateOpenAICost('gpt-audio-mini-2025-12-15', {}, 1000, 500, 200, 100);
+    expect(cost).toBeCloseTo((1000 * 0.6 + 500 * 2.4 + 200 * 10 + 100 * 20) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4', () => {
+    const cost = calculateOpenAICost('gpt-4', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 30 + 500 * 60) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-4.1', () => {
+    const cost = calculateOpenAICost('gpt-4.1', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2 + 500 * 8) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-3.5-turbo', () => {
+    const cost = calculateOpenAICost('gpt-3.5-turbo', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.5 + 500 * 1.5) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o4-mini', () => {
+    const cost = calculateOpenAICost('o4-mini', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.1 + 500 * 4.4) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for codex-mini-latest', () => {
+    const cost = calculateOpenAICost('codex-mini-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.5 + 500 * 6.0) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5', () => {
+    const cost = calculateOpenAICost('gpt-5', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.25 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5-chat-latest', () => {
+    const cost = calculateOpenAICost('gpt-5-chat-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.25 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5-chat', () => {
+    const cost = calculateOpenAICost('gpt-5-chat', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.25 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.1-chat-latest', () => {
+    const cost = calculateOpenAICost('gpt-5.1-chat-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.25 + 500 * 10) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.2-chat-latest', () => {
+    const cost = calculateOpenAICost('gpt-5.2-chat-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.75 + 500 * 14) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.3-chat-latest', () => {
+    const cost = calculateOpenAICost('gpt-5.3-chat-latest', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.75 + 500 * 14) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.5', () => {
+    const cost = calculateOpenAICost('gpt-5.5', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 5 + 500 * 30) / 1e6, 6);
+  });
+
+  it.each([
+    ['gpt-5.6', 5, 30],
+    ['gpt-5.6-sol', 5, 30],
+    ['gpt-5.6-terra', 2.5, 15],
+    ['gpt-5.6-luna', 1, 6],
+  ])('should calculate cost correctly for %s', (model, inputRate, outputRate) => {
+    const cost = calculateOpenAICost(model, {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * inputRate + 500 * outputRate) / 1e6, 6);
+  });
+
+  it.each([
+    ['gpt-5.6', 5, 30, 10, 45],
+    ['gpt-5.6-sol', 5, 30, 10, 45],
+    ['gpt-5.6-terra', 2.5, 15, 5, 22.5],
+    ['gpt-5.6-luna', 1, 6, 2, 9],
+  ])('should apply long-context pricing above 272K for %s', (model, baseInputRate, baseOutputRate, longInputRate, longOutputRate) => {
+    expect(calculateOpenAICost(model, {}, 272_000, 1_000)).toBeCloseTo(
+      (272_000 * baseInputRate + 1_000 * baseOutputRate) / 1e6,
+      6,
+    );
+    expect(calculateOpenAICost(model, {}, 272_001, 1_000)).toBeCloseTo(
+      (272_001 * longInputRate + 1_000 * longOutputRate) / 1e6,
+      6,
+    );
+  });
+
+  it('should calculate long-context cost correctly for gpt-5.5', () => {
+    const cost = calculateOpenAICost('gpt-5.5', {}, 300_000, 1_000);
+    expect(cost).toBeCloseTo((300_000 * 10 + 1_000 * 45) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.5-2026-04-23', () => {
+    const cost = calculateOpenAICost('gpt-5.5-2026-04-23', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 5 + 500 * 30) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4', () => {
+    const cost = calculateOpenAICost('gpt-5.4', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 15) / 1e6, 6);
+  });
+
+  it('should calculate long-context cost correctly for gpt-5.4', () => {
+    const cost = calculateOpenAICost('gpt-5.4', {}, 300_000, 1_000);
+    expect(cost).toBeCloseTo((300_000 * 5 + 1_000 * 22.5) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-2026-03-05', () => {
+    const cost = calculateOpenAICost('gpt-5.4-2026-03-05', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2.5 + 500 * 15) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-mini', () => {
+    const cost = calculateOpenAICost('gpt-5.4-mini', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.75 + 500 * 4.5) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-mini-2026-03-17', () => {
+    const cost = calculateOpenAICost('gpt-5.4-mini-2026-03-17', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.75 + 500 * 4.5) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-nano', () => {
+    const cost = calculateOpenAICost('gpt-5.4-nano', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.2 + 500 * 1.25) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-nano-2026-03-17', () => {
+    const cost = calculateOpenAICost('gpt-5.4-nano-2026-03-17', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.2 + 500 * 1.25) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.2-codex', () => {
+    const cost = calculateOpenAICost('gpt-5.2-codex', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.75 + 500 * 14) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.2-pro', () => {
+    const cost = calculateOpenAICost('gpt-5.2-pro', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 21 + 500 * 168) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.5-pro', () => {
+    const cost = calculateOpenAICost('gpt-5.5-pro', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 30 + 500 * 180) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.5-pro-2026-04-23', () => {
+    const cost = calculateOpenAICost('gpt-5.5-pro-2026-04-23', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 30 + 500 * 180) / 1e6, 6);
+  });
+
+  it('should calculate long-context cost correctly for gpt-5.5-pro', () => {
+    const cost = calculateOpenAICost('gpt-5.5-pro', {}, 300_000, 1_000);
+    expect(cost).toBeCloseTo((300_000 * 60 + 1_000 * 270) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-pro', () => {
+    const cost = calculateOpenAICost('gpt-5.4-pro', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 30 + 500 * 180) / 1e6, 6);
+  });
+
+  it('should calculate long-context cost correctly for gpt-5.4-pro', () => {
+    const cost = calculateOpenAICost('gpt-5.4-pro', {}, 300_000, 1_000);
+    expect(cost).toBeCloseTo((300_000 * 60 + 1_000 * 270) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5.4-pro-2026-03-05', () => {
+    const cost = calculateOpenAICost('gpt-5.4-pro-2026-03-05', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 30 + 500 * 180) / 1e6, 6);
+  });
+
+  it('should recognize GPT-5.5 models with built-in pricing', () => {
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.5')).toBe(true);
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.5-2026-04-23')).toBe(true);
+    expect(OPENAI_RESPONSES_ONLY_MODELS.some((model) => model.id === 'gpt-5.5-pro')).toBe(true);
+    expect(
+      OPENAI_RESPONSES_ONLY_MODELS.some((model) => model.id === 'gpt-5.5-pro-2026-04-23'),
+    ).toBe(true);
+    expect(calculateOpenAICost('gpt-5.5', {}, 1000, 500)).toBeCloseTo(
+      (1000 * 5 + 500 * 30) / 1e6,
+      6,
+    );
+    expect(calculateOpenAICost('gpt-5.5-pro', {}, 1000, 500)).toBeCloseTo(
+      (1000 * 30 + 500 * 180) / 1e6,
+      6,
+    );
+  });
+
+  it('should recognize GPT-5.6 models for Chat Completions and Responses', () => {
+    for (const model of ['gpt-5.6', 'gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna']) {
+      expect(OPENAI_CHAT_MODELS.some((candidate) => candidate.id === model)).toBe(true);
+      expect(OPENAI_RESPONSES_ONLY_MODELS.some((candidate) => candidate.id === model)).toBe(false);
+    }
+  });
+
+  it('should keep GPT-5.4 and GPT-5.5 Pro out of Chat Completions routing', () => {
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.4-pro')).toBe(false);
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.4-pro-2026-03-05')).toBe(false);
+    expect(OPENAI_RESPONSES_ONLY_MODELS.some((model) => model.id === 'gpt-5.4-pro')).toBe(true);
+    expect(
+      OPENAI_RESPONSES_ONLY_MODELS.some((model) => model.id === 'gpt-5.4-pro-2026-03-05'),
+    ).toBe(true);
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.5-pro')).toBe(false);
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-5.5-pro-2026-04-23')).toBe(false);
+  });
+
+  it('should exclude retired preview audio and realtime models from current routing registries', () => {
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-audio-1.5')).toBe(true);
+    expect(OPENAI_CHAT_MODELS.some((model) => model.id === 'gpt-4o-audio-preview')).toBe(false);
+    expect(OPENAI_REALTIME_MODELS.some((model) => model.id === 'gpt-realtime-1.5')).toBe(true);
+    expect(OPENAI_REALTIME_MODELS.some((model) => model.id === 'gpt-realtime-2')).toBe(true);
+    expect(
+      OPENAI_REALTIME_MODELS.some(
+        (model) => model.id === 'gpt-4o-mini-realtime-preview-2024-12-17',
+      ),
+    ).toBe(true);
+    expect(OPENAI_REALTIME_MODELS.some((model) => model.id === 'gpt-4o-realtime-preview')).toBe(
+      false,
+    );
+  });
+
+  it('should calculate cost correctly for gpt-5-nano', () => {
+    const cost = calculateOpenAICost('gpt-5-nano', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.05 + 500 * 0.4) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for gpt-5-mini', () => {
+    const cost = calculateOpenAICost('gpt-5-mini', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 0.25 + 500 * 2) / 1e6, 6);
+  });
+
+  it('should handle undefined token counts', () => {
+    const cost = calculateOpenAICost('gpt-4', {}, undefined, undefined);
+    expect(cost).toBeUndefined();
+  });
+
+  it('should handle unknown models', () => {
+    const cost = calculateOpenAICost('unknown-model', {}, 100, 50);
+    expect(cost).toBeUndefined();
+  });
+
+  it('should use custom cost from config when provided', () => {
+    const cost = calculateOpenAICost('gpt-4', { cost: 0.123 }, 1000, 500);
+    expect(cost).toBe(184.5);
+  });
+
+  it('should use separate custom input and output costs from config when provided', () => {
+    const cost = calculateOpenAICost('gpt-4', { inputCost: 0.001, outputCost: 0.003 }, 1000, 500);
+    expect(cost).toBe(2.5);
+  });
+
+  it('should prefer separate custom input and output costs over custom cost', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4',
+      { cost: 0.123, inputCost: 0.001, outputCost: 0.003 },
+      1000,
+      500,
+    );
+    expect(cost).toBe(2.5);
+  });
+
+  it('should use custom cost as fallback for partial separate text cost overrides', () => {
+    const cost = calculateOpenAICost('gpt-4', { cost: 0.004, outputCost: 0.001 }, 1000, 500);
+    expect(cost).toBe(4.5);
+  });
+
+  it('should calculate cost correctly with custom audioCost', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioCost: 0.05 },
+      1000,
+      500,
+      200,
+      100,
+    );
+    expect(cost).toBe(15.0075);
+  });
+
+  it('should use separate custom audio input and output costs from config when provided', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioInputCost: 0.01, audioOutputCost: 0.03 },
+      1000,
+      500,
+      200,
+      100,
+    );
+    expect(cost).toBe(5.0075);
+  });
+
+  it('should prefer separate custom audio costs over custom audioCost', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioCost: 0.05, audioInputCost: 0.01, audioOutputCost: 0.03 },
+      1000,
+      500,
+      200,
+      100,
+    );
+    expect(cost).toBe(5.0075);
+  });
+
+  it('should fall back to model default for partial audio override (audioInputCost only)', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioInputCost: 0.01 },
+      1000,
+      500,
+      200,
+      100,
+    );
+    const expected = (2.5 * 1000) / 1e6 + (10 * 500) / 1e6 + 0.01 * 200 + (80 * 100) / 1e6;
+    expect(cost).toBeCloseTo(expected, 6);
+  });
+
+  it('should fall back to audioCost for partial audio override (audioInputCost + audioCost)', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioCost: 0.02, audioInputCost: 0.01 },
+      1000,
+      500,
+      200,
+      100,
+    );
+    const expected = (2.5 * 1000) / 1e6 + (10 * 500) / 1e6 + 0.01 * 200 + 0.02 * 100;
+    expect(cost).toBeCloseTo(expected, 6);
+  });
+
+  it('should handle a model with no cost property', () => {
+    const cost = calculateOpenAICost('text-davinci-002', {}, 1000, 500);
+    expect(cost).toBeUndefined();
+  });
+
+  it('should calculate cost correctly for o1-pro', () => {
+    const cost = calculateOpenAICost('o1-pro', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 150 + 500 * 600) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3-pro', () => {
+    const cost = calculateOpenAICost('o3-pro', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 20 + 500 * 80) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o1-pro-2025-03-19', () => {
+    const cost = calculateOpenAICost('o1-pro-2025-03-19', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 150 + 500 * 600) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3', () => {
+    const cost = calculateOpenAICost('o3', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2 + 500 * 8) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3-2025-04-16', () => {
+    const cost = calculateOpenAICost('o3-2025-04-16', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2 + 500 * 8) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3-pro-2025-06-10', () => {
+    const cost = calculateOpenAICost('o3-pro-2025-06-10', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.06); // 20/1M * 1000 + 80/1M * 500
+  });
+
+  it('should calculate audio token costs for gpt-4o-realtime-preview-2024-12-17', () => {
+    const cost = calculateOpenAICost('gpt-4o-realtime-preview-2024-12-17', {}, 1000, 500, 200, 100);
+    const expectedCost = (1000 * 5 + 500 * 20 + 200 * 40 + 100 * 80) / 1e6;
+    expect(cost).toBeCloseTo(expectedCost, 6);
+  });
+
+  it('should calculate audio token costs for gpt-4o-mini-realtime-preview-2024-12-17', () => {
+    const cost = calculateOpenAICost(
+      'gpt-4o-mini-realtime-preview-2024-12-17',
+      {},
+      1000,
+      500,
+      200,
+      100,
+    );
+    const expectedCost = (1000 * 0.6 + 500 * 2.4 + 200 * 10 + 100 * 20) / 1e6;
+    expect(cost).toBeCloseTo(expectedCost, 6);
+  });
+
+  it('should calculate audio token costs for gpt-realtime', () => {
+    const cost = calculateOpenAICost('gpt-realtime', {}, 1000, 500, 200, 100);
+    const expectedCost = (1000 * 4 + 500 * 16 + 200 * 32 + 100 * 64) / 1e6;
+    expect(cost).toBeCloseTo(expectedCost, 6);
+  });
+
+  it('should calculate audio token costs for gpt-realtime-2', () => {
+    const cost = calculateOpenAICost('gpt-realtime-2', {}, 1000, 500, 200, 100);
+    const expectedCost = (1000 * 4 + 500 * 24 + 200 * 32 + 100 * 64) / 1e6;
+    expect(cost).toBeCloseTo(expectedCost, 6);
+  });
+
+  it('should calculate audio token costs for gpt-realtime-mini-2025-12-15', () => {
+    const cost = calculateOpenAICost('gpt-realtime-mini-2025-12-15', {}, 1000, 500, 200, 100);
+    const expectedCost = (1000 * 0.6 + 500 * 2.4 + 200 * 10 + 100 * 20) / 1e6;
+    expect(cost).toBeCloseTo(expectedCost, 6);
+  });
+
+  it('should return zero cost for zero tokens', () => {
+    const cost = calculateOpenAICost('gpt-4.1', {}, 0, 0);
+    expect(cost).toBe(0);
+  });
+
+  it('should handle only prompt tokens', () => {
+    const cost = calculateOpenAICost('gpt-4.1', {}, 1000, 0);
+    expect(cost).toBeCloseTo((1000 * 2) / 1e6, 6);
+  });
+
+  it('should handle only completion tokens', () => {
+    const cost = calculateOpenAICost('gpt-4.1', {}, 0, 1000);
+    expect(cost).toBeCloseTo((1000 * 8) / 1e6, 6);
+  });
+
+  it('should handle large token counts', () => {
+    const cost = calculateOpenAICost('gpt-4.1', {}, 1000000, 1000000);
+    expect(cost).toBeCloseTo((1000000 * 2 + 1000000 * 8) / 1e6, 6);
+  });
+
+  it('should handle mixed undefined audio tokens', () => {
+    const cost = calculateOpenAICost('gpt-4o-audio-preview', {}, 1000, 500, undefined, 100);
+    expect(cost).toBeUndefined();
+  });
+
+  it('should use custom audioCost from config when provided', () => {
+    const audioCost = 0.05; // per 1M tokens
+
+    const promptTokens = 1000;
+    const completionTokens = 500;
+    const audioPromptTokens = 200;
+    const audioCompletionTokens = 100;
+
+    const model = OPENAI_CHAT_MODELS.find(
+      (m) =>
+        m.id === 'gpt-4o-audio-preview' &&
+        m.cost &&
+        'audioInput' in m.cost &&
+        'audioOutput' in m.cost,
+    );
+
+    if (!model || !model.cost) {
+      return;
+    }
+
+    const baseInputCost = model.cost.input * promptTokens;
+    const baseOutputCost = model.cost.output * completionTokens;
+
+    const audioInputCostCustom = audioCost * audioPromptTokens;
+    const audioOutputCostCustom = audioCost * audioCompletionTokens;
+
+    const expectedTotalCost =
+      (baseInputCost + baseOutputCost + audioInputCostCustom + audioOutputCostCustom) / 1;
+
+    const cost = calculateOpenAICost(
+      'gpt-4o-audio-preview',
+      { audioCost },
+      promptTokens,
+      completionTokens,
+      audioPromptTokens,
+      audioCompletionTokens,
+    );
+
+    expect(cost).toBeCloseTo(expectedTotalCost, 2);
+  });
+
+  it('should handle a non-existent model with no cost property', () => {
+    const fakeModelName = 'non-existent-model-with-no-cost';
+
+    const cost = calculateOpenAICost(fakeModelName, {}, 1000, 500);
+    expect(cost).toBeUndefined();
+  });
+
+  // Legacy GPT-4 model tests
+  it('should calculate cost correctly for gpt-4-0314', () => {
+    const cost = calculateOpenAICost('gpt-4-0314', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.06); // 30/1M * 1000 + 60/1M * 500
+  });
+
+  it('should calculate cost correctly for gpt-4-32k-0314', () => {
+    const cost = calculateOpenAICost('gpt-4-32k-0314', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.12); // 60/1M * 1000 + 120/1M * 500
+  });
+
+  it('should calculate cost correctly for gpt-4-32k-0613', () => {
+    const cost = calculateOpenAICost('gpt-4-32k-0613', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.12); // 60/1M * 1000 + 120/1M * 500
+  });
+
+  it('should calculate cost correctly for gpt-4-vision-preview', () => {
+    const cost = calculateOpenAICost('gpt-4-vision-preview', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.025); // 10/1M * 1000 + 30/1M * 500
+  });
+
+  // Legacy GPT-3.5 model tests
+  it('should calculate cost correctly for gpt-3.5-turbo-0301', () => {
+    const cost = calculateOpenAICost('gpt-3.5-turbo-0301', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.0025); // 1.5/1M * 1000 + 2/1M * 500
+  });
+
+  it('should calculate cost correctly for gpt-3.5-turbo-16k', () => {
+    const cost = calculateOpenAICost('gpt-3.5-turbo-16k', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.005); // 3/1M * 1000 + 4/1M * 500
+  });
+
+  it('should calculate cost correctly for gpt-3.5-turbo-16k-0613', () => {
+    const cost = calculateOpenAICost('gpt-3.5-turbo-16k-0613', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.005); // 3/1M * 1000 + 4/1M * 500
+  });
+
+  // Latest audio model test
+  it('should calculate cost correctly for gpt-4o-audio-preview-2025-06-03', () => {
+    const cost = calculateOpenAICost('gpt-4o-audio-preview-2025-06-03', {}, 1000, 500);
+    expect(cost).toBeCloseTo(0.0075); // 2.5/1M * 1000 + 10/1M * 500
+  });
+
+  it('should calculate cost correctly for o4-mini (responses model)', () => {
+    const cost = calculateOpenAICost('o4-mini', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 1.1 + 500 * 4.4) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3-deep-research', () => {
+    const cost = calculateOpenAICost('o3-deep-research', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 10 + 500 * 40) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o3-deep-research-2025-06-26', () => {
+    const cost = calculateOpenAICost('o3-deep-research-2025-06-26', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 10 + 500 * 40) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o4-mini-deep-research', () => {
+    const cost = calculateOpenAICost('o4-mini-deep-research', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2 + 500 * 8) / 1e6, 6);
+  });
+
+  it('should calculate cost correctly for o4-mini-deep-research-2025-06-26', () => {
+    const cost = calculateOpenAICost('o4-mini-deep-research-2025-06-26', {}, 1000, 500);
+    expect(cost).toBeCloseTo((1000 * 2 + 500 * 8) / 1e6, 6);
+  });
+});
+
+describe('validateFunctionCall', () => {
+  const sampleFunction = {
+    name: 'testFunction',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number' },
+      },
+      required: ['foo'],
+    },
+  };
+
+  it('should validate valid function call', () => {
+    const functionCall = {
+      name: 'testFunction',
+      arguments: JSON.stringify({ foo: 'test', bar: 123 }),
+    };
+
+    expect(() => validateFunctionCall(functionCall, [sampleFunction], {})).not.toThrow();
+  });
+
+  it('should throw error for invalid function name', () => {
+    const functionCall = {
+      name: 'nonexistentFunction',
+      arguments: JSON.stringify({ foo: 'test' }),
+    };
+
+    expect(() => validateFunctionCall(functionCall, [sampleFunction], {})).toThrow(
+      'Called "nonexistentFunction", but there is no function with that name',
+    );
+  });
+
+  it('should throw error for invalid arguments', () => {
+    const functionCall = {
+      name: 'testFunction',
+      arguments: JSON.stringify({ bar: 'not a number' }),
+    };
+
+    expect(() => validateFunctionCall(functionCall, [sampleFunction], {})).toThrow(
+      /Call to "testFunction" does not match schema/,
+    );
+  });
+});
+
+describe('formatOpenAiError', () => {
+  it('should format error with type and code', () => {
+    const error = {
+      error: {
+        message: 'Error message',
+        type: 'error_type',
+        code: 'error_code',
+      },
+    };
+
+    const result = formatOpenAiError(error);
+    expect(result).toContain('API error: Error message');
+    expect(result).toContain('Type: error_type');
+    expect(result).toContain('Code: error_code');
+  });
+
+  it('should format error without type and code', () => {
+    const error = {
+      error: {
+        message: 'Error message',
+      },
+    };
+
+    const result = formatOpenAiError(error);
+    expect(result).toContain('API error: Error message');
+    expect(result).not.toContain('Type:');
+    expect(result).not.toContain('Code:');
+  });
+});