promptfoo--promptfoo/test/providers/openai-codex-sdk.e2e.test.ts

import { mockProcessEnv } from '../util/utils';

/**
 * End-to-End tests for OpenAI Codex SDK Provider
 * These tests use the real @openai/codex-sdk package (must be installed)
 *
 * Requirements:
 * - @openai/codex-sdk package installed
 * - CODEX_API_KEY, CODEX_E2E_API_KEY, or OPENAI_API_KEY environment variable
 * - Access to Codex-compatible models (gpt-5.2, gpt-5.1-codex, etc.)
 *
 * Run with:
 *   CODEX_API_KEY=... npx vitest run openai-codex-sdk.e2e
 *
 * Model configuration:
 *   Set CODEX_E2E_MODEL to override the default model (gpt-5.2).
 *   Example: CODEX_E2E_MODEL=gpt-5.1-codex npx vitest run openai-codex-sdk.e2e
 *
 * Note: These models may require specific API access tiers or partnerships.
 * Tests will be skipped automatically if prerequisites are not met.
 */

import fs from 'fs';
import path from 'path';

import { afterAll, describe, expect, it, vi } from 'vitest';

vi.unmock('@openai/codex-sdk');
vi.unmock('../../src/esm');

const hasSdk = fs.existsSync(
  path.resolve(process.cwd(), 'node_modules/@openai/codex-sdk/package.json'),
);
const openAiApiKey =
  process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY !== 'test-openai-api-key'
    ? process.env.OPENAI_API_KEY
    : undefined;
const e2eApiKey = process.env.CODEX_E2E_API_KEY || process.env.CODEX_API_KEY || openAiApiKey;
const originalOpenAiApiKey = process.env.OPENAI_API_KEY;
const originalCodexApiKey = process.env.CODEX_API_KEY;

// vitest.setup.ts installs a dummy OPENAI_API_KEY for unit tests. Ignore that value for E2E runs,
// and only restore a real key when one is explicitly provided.
if (e2eApiKey) {
  mockProcessEnv({ OPENAI_API_KEY: e2eApiKey });
  mockProcessEnv({ CODEX_API_KEY: e2eApiKey });
}

afterAll(() => {
  if (originalOpenAiApiKey === undefined) {
    mockProcessEnv({ OPENAI_API_KEY: undefined });
  } else {
    mockProcessEnv({ OPENAI_API_KEY: originalOpenAiApiKey });
  }

  if (originalCodexApiKey === undefined) {
    mockProcessEnv({ CODEX_API_KEY: undefined });
  } else {
    mockProcessEnv({ CODEX_API_KEY: originalCodexApiKey });
  }
});

import { OpenAICodexSDKProvider } from '../../src/providers/openai/codex-sdk';

// Default model for E2E tests - override with CODEX_E2E_MODEL environment variable
const DEFAULT_E2E_MODEL = process.env.CODEX_E2E_MODEL || 'gpt-5.2';

describe('OpenAICodexSDKProvider E2E', () => {
  const hasApiKey = !!e2eApiKey;
  const testTimeout = 60000; // 60 seconds for real API calls

  // Skip all tests if no API key or SDK not installed
  const describeOrSkip = hasApiKey && hasSdk ? describe : describe.skip;

  describeOrSkip('Real SDK Integration', () => {
    it(
      'should successfully generate code with real SDK',
      async () => {
        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: process.cwd(),
            skip_git_repo_check: false,
          },
        });

        const response = await provider.callApi(
          'Write a Python function that calculates fibonacci numbers. Output only the code.',
        );

        expect(response.error).toBeUndefined();
        expect(response.output).toBeTruthy();
        expect(response.output).toContain('def');
        expect(response.output).toContain('fibonacci');
        expect(response.tokenUsage).toBeDefined();
        expect(response.tokenUsage?.total).toBeGreaterThan(0);
        expect(response.sessionId).toBeTruthy();
      },
      testTimeout,
    );

    it(
      'should handle structured output with JSON schema',
      async () => {
        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: process.cwd(),
            skip_git_repo_check: false,
            output_schema: {
              type: 'object',
              properties: {
                function_name: { type: 'string' },
                parameters: {
                  type: 'array',
                  items: { type: 'string' },
                },
                return_type: { type: 'string' },
              },
              required: ['function_name', 'parameters', 'return_type'],
            },
          },
        });

        const response = await provider.callApi(
          'Describe the factorial function signature in the requested format.',
        );

        expect(response.error).toBeUndefined();
        expect(response.output).toBeTruthy();

        // Should return valid JSON
        const parsed = JSON.parse(response.output as string);
        expect(parsed.function_name).toBeTruthy();
        expect(Array.isArray(parsed.parameters)).toBe(true);
        expect(parsed.return_type).toBeTruthy();
      },
      testTimeout,
    );

    it(
      'should reuse threads with persist_threads',
      async () => {
        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: process.cwd(),
            skip_git_repo_check: false,
            persist_threads: true,
          },
        });

        // First call
        const response1 = await provider.callApi('What is 2 + 2?');
        expect(response1.error).toBeUndefined();
        const sessionId1 = response1.sessionId;

        // Second call with same config - should reuse thread
        const response2 = await provider.callApi('What is 3 + 3?');
        expect(response2.error).toBeUndefined();
        const sessionId2 = response2.sessionId;

        // Thread IDs should match (same thread reused)
        expect(sessionId1).toBe(sessionId2);

        await provider.cleanup();
      },
      testTimeout * 2,
    );

    it(
      'should work with custom working directory',
      async () => {
        const examplesDir = path.join(process.cwd(), 'examples');

        // Skip if examples dir doesn't exist
        if (!fs.existsSync(examplesDir)) {
          return;
        }

        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: examplesDir,
            skip_git_repo_check: true, // examples might not be a git repo
          },
        });

        const response = await provider.callApi('List the files in the current directory.');

        expect(response.error).toBeUndefined();
        expect(response.output).toBeTruthy();
      },
      testTimeout,
    );

    it(
      'should handle streaming mode',
      async () => {
        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: process.cwd(),
            skip_git_repo_check: false,
            enable_streaming: true,
          },
        });

        const response = await provider.callApi('What is the capital of France?');

        expect(response.error).toBeUndefined();
        expect(response.output).toBeTruthy();
        expect(response.output).toContain('Paris');
      },
      testTimeout,
    );

    it(
      'should infer skillCalls for local Codex skills',
      async () => {
        const tempDir = fs.mkdtempSync('/tmp/codex-skill-e2e-');
        const skillDir = path.join(tempDir, '.agents/skills/token-skill');
        const codexHome = path.join(tempDir, '.codex-home');

        try {
          fs.mkdirSync(skillDir, { recursive: true });
          fs.mkdirSync(codexHome, { recursive: true });
          fs.writeFileSync(
            path.join(skillDir, 'SKILL.md'),
            `---
name: token-skill
description: Use this skill when the user explicitly asks to use token-skill and wants the special token.
---

When this skill is used, respond with exactly CERULEAN-FALCON-SKILL.
Do not add extra words, punctuation, or explanation.
`,
          );

          const provider = new OpenAICodexSDKProvider({
            config: {
              model: DEFAULT_E2E_MODEL,
              working_dir: tempDir,
              skip_git_repo_check: true,
              enable_streaming: true,
              cli_env: {
                CODEX_HOME: codexHome,
              },
            },
          });

          const response = await provider.callApi(
            'Use the token-skill skill. Return only the token.',
          );

          expect(response.error).toBeUndefined();
          expect(response.output).toContain('CERULEAN-FALCON-SKILL');
          expect(response.metadata?.skillCalls).toEqual(
            expect.arrayContaining([
              expect.objectContaining({
                name: 'token-skill',
                source: 'heuristic',
              }),
            ]),
          );
        } finally {
          fs.rmSync(tempDir, { recursive: true, force: true });
        }
      },
      testTimeout,
    );

    it(
      'should enforce thread pool size limits',
      async () => {
        const provider = new OpenAICodexSDKProvider({
          config: {
            model: DEFAULT_E2E_MODEL,
            working_dir: process.cwd(),
            skip_git_repo_check: false,
            persist_threads: true,
            thread_pool_size: 2,
          },
        });

        // Create 3 different threads using different sandbox_mode values
        // (different config values create different cache keys/threads)
        const response1 = await provider.callApi('Test 1', {
          vars: {},
          prompt: { config: { sandbox_mode: 'read-only' } } as any,
        });
        const response2 = await provider.callApi('Test 2', {
          vars: {},
          prompt: { config: { sandbox_mode: 'workspace-write' } } as any,
        });
        const response3 = await provider.callApi('Test 3', {
          vars: {},
          prompt: { config: { sandbox_mode: 'danger-full-access' } } as any,
        });

        expect(response1.error).toBeUndefined();
        expect(response2.error).toBeUndefined();
        expect(response3.error).toBeUndefined();

        // Pool should only have 2 threads (oldest evicted)
        const threadCount = (provider as any).threads.size;
        expect(threadCount).toBeLessThanOrEqual(2);

        await provider.cleanup();
      },
      testTimeout * 3,
    );
  });

  describeOrSkip('Error Handling', () => {
    it('should handle invalid API key gracefully', async () => {
      const provider = new OpenAICodexSDKProvider({
        config: {
          apiKey: 'invalid-key',
          working_dir: process.cwd(),
          skip_git_repo_check: false,
        },
      });

      const response = await provider.callApi('Test');

      expect(response.error).toBeDefined();
      expect(typeof response.error).toBe('string');
    });

    it('should validate Git repository requirement', async () => {
      const tempDir = fs.mkdtempSync('/tmp/codex-test-');

      try {
        const provider = new OpenAICodexSDKProvider({
          config: {
            working_dir: tempDir,
            skip_git_repo_check: false,
          },
        });

        const response = await provider.callApi('Test');

        expect(response.error).toMatch(/is not inside a Git repository/);
      } finally {
        fs.rmSync(tempDir, { recursive: true, force: true });
      }
    });
  });
});