promptfoo--promptfoo/test/providers/openai/responses/reasoning.test.ts

// Load-bearing: registers shared vi.mock / beforeEach hooks before any
// module-under-test import below. See ./setup.ts for details.
import './setup';

import { describe, expect, it, vi } from 'vitest';
import * as cache from '../../../../src/cache';
import { OpenAiResponsesProvider } from '../../../../src/providers/openai/responses';
import { LONG_RUNNING_MODEL_TIMEOUT_MS } from '../../../../src/providers/shared';
import { setOpenAiEnv } from './setup';
import type { Mock } from 'vitest';

describe('OpenAiResponsesProvider reasoning models', () => {
  it('should prefer OPENAI_MAX_COMPLETION_TOKENS over OPENAI_MAX_TOKENS for reasoning models', async () => {
    setOpenAiEnv({
      OPENAI_MAX_COMPLETION_TOKENS: '4096',
      OPENAI_MAX_TOKENS: '2048',
    });

    const provider = new OpenAiResponsesProvider('o1-preview', {
      config: { apiKey: 'test-key' },
    });

    const { body } = await provider.getOpenAiBody('Test prompt');
    expect(body.max_output_tokens).toBe(4096);
  });

  it('should fall back to OPENAI_MAX_TOKENS for reasoning models when OPENAI_MAX_COMPLETION_TOKENS is unset', async () => {
    setOpenAiEnv({ OPENAI_MAX_TOKENS: '2048' });

    const provider = new OpenAiResponsesProvider('o1-preview', {
      config: { apiKey: 'test-key' },
    });

    const { body } = await provider.getOpenAiBody('Test prompt');
    expect(body.max_output_tokens).toBe(2048);
  });

  it('should not apply a hardcoded max_output_tokens default for reasoning models', async () => {
    const provider = new OpenAiResponsesProvider('o1-preview', {
      config: { apiKey: 'test-key' },
    });

    const { body } = await provider.getOpenAiBody('Test prompt');
    expect(body.max_output_tokens).toBeUndefined();
    expect('max_output_tokens' in body).toBe(false);
  });

  it('should handle reasoning models correctly', async () => {
    // Mock API response for o1-pro model
    const mockApiResponse = {
      id: 'resp_abc123',
      object: 'response',
      created_at: 1234567890,
      status: 'completed',
      model: 'o1-pro',
      output: [
        {
          type: 'message',
          id: 'msg_abc123',
          status: 'completed',
          role: 'assistant',
          content: [
            {
              type: 'output_text',
              text: 'This is a response from o1-pro',
            },
          ],
        },
      ],
      usage: {
        input_tokens: 15,
        output_tokens: 30,
        output_tokens_details: {
          reasoning_tokens: 100,
        },
        total_tokens: 45,
      },
    };

    // Setup mock for fetchWithCache
    vi.mocked(cache.fetchWithCache).mockResolvedValue({
      data: mockApiResponse,
      cached: false,
      status: 200,
      statusText: 'OK',
    });

    // Initialize the provider with reasoning model settings
    const provider = new OpenAiResponsesProvider('o1-pro', {
      config: {
        apiKey: 'test-key',
        reasoning_effort: 'medium',
        max_completion_tokens: 2000,
      },
    });

    // Call the API
    const result = await provider.callApi('Test prompt');

    // Verify the request body includes reasoning effort
    expect(cache.fetchWithCache).toHaveBeenCalledWith(
      expect.any(String),
      expect.objectContaining({
        body: expect.stringContaining('"reasoning":{"effort":"medium"}'),
      }),
      expect.any(Number),
      'json',
      undefined,
      undefined,
    );

    // Assertions
    expect(result.error).toBeUndefined();
    expect(result.output).toBe('This is a response from o1-pro');
    // Just test that the total tokens is present, but don't test for reasoning tokens
    // as the implementation may handle these details differently
    expect(result.tokenUsage?.total).toBe(45);
  });

  it.each([
    { model: 'gpt-5.6', reasoningEffort: 'max', maxOutputTokens: 4000 },
    { model: 'gpt-5.6-sol', reasoningEffort: 'max', maxOutputTokens: 4000 },
    { model: 'gpt-5.6-terra', reasoningEffort: 'max', maxOutputTokens: 4000 },
    { model: 'gpt-5.6-luna', reasoningEffort: 'max', maxOutputTokens: 4000 },
    { model: 'o3', reasoningEffort: 'high', maxOutputTokens: 2000 },
    { model: 'o3-pro', reasoningEffort: 'high', maxOutputTokens: 2000 },
    { model: 'o4-mini', reasoningEffort: 'medium', maxOutputTokens: 1000 },
    { model: 'codex-mini-latest', reasoningEffort: 'medium', maxOutputTokens: 1000 },
  ] as const)('should configure $model model correctly with reasoning parameters', async ({
    model,
    reasoningEffort,
    maxOutputTokens,
  }) => {
    const mockApiResponse = {
      id: 'resp_abc123',
      status: 'completed',
      model,
      output: [
        {
          type: 'message',
          role: 'assistant',
          content: [
            {
              type: 'output_text',
              text: `Response from ${model} model`,
            },
          ],
        },
      ],
      usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
    };

    vi.mocked(cache.fetchWithCache).mockResolvedValue({
      data: mockApiResponse,
      cached: false,
      status: 200,
      statusText: 'OK',
    });

    const provider = new OpenAiResponsesProvider(model, {
      config: {
        apiKey: 'test-key',
        reasoning_effort: reasoningEffort,
        max_output_tokens: maxOutputTokens,
      },
    });

    await provider.callApi('Test prompt');

    const mockCall = vi.mocked(cache.fetchWithCache).mock.calls[0];
    const reqOptions = mockCall[1] as { body: string };
    const body = JSON.parse(reqOptions.body);

    expect(body.model).toBe(model);
    expect(body.reasoning).toEqual({ effort: reasoningEffort });
    expect(body.max_output_tokens).toBe(maxOutputTokens);
    expect(body.temperature).toBeUndefined();
  });

  it('should forward GPT-5.6 persisted reasoning and Pro mode', async () => {
    const provider = new OpenAiResponsesProvider('gpt-5.6-sol', {
      config: {
        reasoning: { effort: 'max', context: 'all_turns', mode: 'pro' },
      },
    });

    const { body } = await provider.getOpenAiBody('Test prompt');

    expect(body.reasoning).toEqual({ effort: 'max', context: 'all_turns', mode: 'pro' });
  });

  describe('deep research model validation', () => {
    it('should require web_search_preview tool for deep research models', async () => {
      const provider = new OpenAiResponsesProvider('o3-deep-research', {
        config: {
          apiKey: 'test-key',
          tools: [{ type: 'code_interpreter' } as any], // Missing web_search_preview
        },
      });

      const result = await provider.callApi('Test prompt');
      expect(result.error).toContain('requires the web_search_preview tool');
      expect(result.error).toContain('o3-deep-research');
      expect(cache.fetchWithCache).not.toHaveBeenCalled();
    });

    it('should accept deep research models with web_search_preview tool', async () => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Research complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider('o4-mini-deep-research', {
        config: {
          apiKey: 'test-key',
          tools: [{ type: 'web_search_preview' } as any],
        },
      });

      const result = await provider.callApi('Test prompt');
      expect(result.error).toBeUndefined();
      expect(result.output).toBe('Research complete');
    });

    it('should require MCP tools to have require_approval: never for deep research', async () => {
      const provider = new OpenAiResponsesProvider('o3-deep-research', {
        config: {
          apiKey: 'test-key',
          tools: [
            { type: 'web_search_preview' } as any,
            {
              type: 'mcp',
              server_label: 'test_server',
              server_url: 'http://test.com',
              require_approval: 'auto' as any, // Should be 'never'
            },
          ],
        },
      });

      const result = await provider.callApi('Test prompt');
      expect(result.error).toContain("requires MCP tools to have require_approval: 'never'");
      expect(cache.fetchWithCache).not.toHaveBeenCalled();
    });

    it('should use longer timeout for deep research models', async () => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Research complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider('o3-deep-research', {
        config: {
          apiKey: 'test-key',
          tools: [{ type: 'web_search_preview' } as any],
        },
      });

      await provider.callApi('Test prompt');

      // Check that fetchWithCache was called with 10-minute timeout
      expect(cache.fetchWithCache).toHaveBeenCalledWith(
        expect.any(String),
        expect.any(Object),
        LONG_RUNNING_MODEL_TIMEOUT_MS,
        'json',
        undefined,
        undefined,
      );
    });

    it('should use longer timeout for gpt-5-pro models', async () => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Response complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider('gpt-5-pro', {
        config: {
          apiKey: 'test-key',
        },
      });

      await provider.callApi('Test prompt');

      // Check that fetchWithCache was called with 10-minute timeout
      expect(cache.fetchWithCache).toHaveBeenCalledWith(
        expect.any(String),
        expect.any(Object),
        LONG_RUNNING_MODEL_TIMEOUT_MS,
        'json',
        undefined,
        undefined,
      );
    });

    it('should use longer timeout for gpt-5.2-pro models', async () => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Response complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider('gpt-5.2-pro', {
        config: {
          apiKey: 'test-key',
        },
      });

      await provider.callApi('Test prompt');

      expect(cache.fetchWithCache).toHaveBeenCalledWith(
        expect.any(String),
        expect.any(Object),
        LONG_RUNNING_MODEL_TIMEOUT_MS,
        'json',
        undefined,
        undefined,
      );
    });

    it.each([
      'gpt-5.4-pro',
      'gpt-5.5-pro',
    ])('should use longer timeout for %s models', async (model) => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Response complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider(model, {
        config: {
          apiKey: 'test-key',
        },
      });

      await provider.callApi('Test prompt');

      expect(cache.fetchWithCache).toHaveBeenCalledWith(
        expect.any(String),
        expect.any(Object),
        LONG_RUNNING_MODEL_TIMEOUT_MS,
        'json',
        undefined,
        undefined,
      );
    });

    it('should use longer timeout for gpt-5.5-pro models', async () => {
      const mockData = {
        output: [
          {
            type: 'message',
            role: 'assistant',
            content: [{ type: 'output_text', text: 'Response complete' }],
          },
        ],
        usage: { input_tokens: 100, output_tokens: 200 },
      };

      (cache.fetchWithCache as Mock).mockResolvedValueOnce({
        data: mockData,
        status: 200,
        statusText: 'OK',
        cached: false,
      });

      const provider = new OpenAiResponsesProvider('gpt-5.5-pro', {
        config: {
          apiKey: 'test-key',
        },
      });

      await provider.callApi('Test prompt');

      expect(cache.fetchWithCache).toHaveBeenCalledWith(
        expect.any(String),
        expect.any(Object),
        LONG_RUNNING_MODEL_TIMEOUT_MS,
        'json',
        undefined,
        undefined,
      );
    });

    it('should handle reasoning items with empty summary correctly', async () => {
      const mockData = {
        output: [
          {
            type: 'reasoning',
            summary: [], // Empty array (edge case)
          },
          {
            type: 'message',
            role: 'assistant',
            content: [
              {
                type: 'output_text',
                text: 'Final answer',
              },
            ],
          },
        ],
        usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
      };

      vi.mocked(cache.fetchWithCache).mockResolvedValue({
        data: mockData,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const provider = new OpenAiResponsesProvider('o3-deep-research', {
        config: {
          apiKey: 'test-key',
          tools: [{ type: 'web_search_preview' } as any],
        },
      });

      const result = await provider.callApi('Test prompt');

      // Should only include the valid reasoning summary and final answer
      expect(result.error).toBeUndefined();
      expect(result.output).toBe('Final answer');
      expect(result.output).not.toContain('Reasoning: []');
    });
  });

  it('should handle reasoning items with summary correctly', async () => {
    const mockData = {
      output: [
        {
          type: 'reasoning',
          summary: [{ type: 'summary_text', text: 'Valid reasoning summary' }],
        },
        {
          type: 'message',
          role: 'assistant',
          content: [{ type: 'output_text', text: 'Final answer' }],
        },
      ],
      usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 },
    };

    vi.mocked(cache.fetchWithCache).mockResolvedValue({
      data: mockData,
      cached: false,
      status: 200,
      statusText: 'OK',
    });

    const provider = new OpenAiResponsesProvider('gpt-4o', {
      config: {
        apiKey: 'test-key',
      },
    });

    const result = await provider.callApi('Test prompt');
    expect(result.error).toBeUndefined();
    expect(result.output).toBe('Reasoning: Valid reasoning summary\nFinal answer');
  });
});