promptfoo--promptfoo/test/providers.test.ts

import fs from 'fs';
import path from 'path';

import { beforeEach, describe, expect, it, vi } from 'vitest';
import cliState from '../src/cliState';
import { CLOUD_PROVIDER_PREFIX } from '../src/constants';
import { HttpProvider } from '../src/providers/http';
import { loadApiProvider, loadApiProviders } from '../src/providers/index';
import { OpenAiChatCompletionProvider } from '../src/providers/openai/chat';
import { OpenAICodexSDKProvider } from '../src/providers/openai/codex-sdk';
import { OpenAiEmbeddingProvider } from '../src/providers/openai/embedding';
import { OpenAiRealtimeProvider } from '../src/providers/openai/realtime';
import { OpenAiResponsesProvider } from '../src/providers/openai/responses';
import { PythonProvider } from '../src/providers/pythonCompletion';
import { ScriptCompletionProvider } from '../src/providers/scriptCompletion';
import { WebSocketProvider } from '../src/providers/websocket';
import { getCloudDatabaseId, getProviderFromCloud, isCloudProvider } from '../src/util/cloud';
import * as fileUtil from '../src/util/file';
import { loadYaml } from '../src/util/yamlLoad';
import { mockProcessEnv } from './util/utils';

import type { ProviderOptions } from '../src/types/index';

vi.mock('fs');
vi.mock('../src/util/yamlLoad');
vi.mock('../src/util/fetch/index.ts');
vi.mock('../src/providers/http');
vi.mock('../src/providers/openai/chat');
vi.mock('../src/providers/openai/embedding');
vi.mock('../src/providers/openai/realtime');
vi.mock('../src/providers/openai/responses');
vi.mock('../src/providers/pythonCompletion');
vi.mock('../src/providers/scriptCompletion');
vi.mock('../src/providers/websocket');
vi.mock('../src/util/cloud');
vi.mock('../src/util/file', async () => {
  const actual = await vi.importActual<typeof import('../src/util/file')>('../src/util/file');
  return {
    ...actual,
    maybeLoadConfigFromExternalFile: vi.fn((input) => input),
  };
});

describe('loadApiProvider', () => {
  beforeEach(() => {
    vi.resetAllMocks();
    vi.spyOn(process, 'exit').mockImplementation((() => {}) as any);

    // Mock the cloud utility functions
    vi.mocked(isCloudProvider).mockImplementation((path: string) =>
      path.startsWith('promptfoo://provider/'),
    );
    vi.mocked(getCloudDatabaseId).mockImplementation((path: string) =>
      path.slice('promptfoo://provider/'.length),
    );

    // Reset maybeLoadConfigFromExternalFile mock to default implementation
    vi.mocked(fileUtil.maybeLoadConfigFromExternalFile).mockImplementation((input: any) => input);
  });

  it('should load echo provider', async () => {
    const provider = await loadApiProvider('echo');
    expect(provider.id()).toBe('echo');
    await expect(provider.callApi('test')).resolves.toEqual({
      output: 'test',
      raw: 'test',
      cost: 0,
      cached: false,
      isRefusal: false,
      tokenUsage: {
        total: 0,
        prompt: 0,
        completion: 0,
        numRequests: 1,
      },
      metadata: {},
    });
  });

  it('should load file provider from yaml', async () => {
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'test-key',
        temperature: 0.7,
      },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const provider = await loadApiProvider('file://test.yaml', {
      basePath: '/test',
    });

    expect(fs.readFileSync).toHaveBeenCalledWith(path.join('/test', 'test.yaml'), 'utf8');
    expect(loadYaml).toHaveBeenCalledWith('yaml content');
    expect(provider).toBeDefined();
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', expect.any(Object));
  });

  it('should load file provider from json', async () => {
    const jsonContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'test-key',
      },
    };
    vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(jsonContent));
    vi.mocked(loadYaml).mockReturnValue(jsonContent);

    const provider = await loadApiProvider('file://test.json', {
      basePath: '/test',
    });

    expect(fs.readFileSync).toHaveBeenCalledWith(path.join('/test', 'test.json'), 'utf8');
    expect(loadYaml).toHaveBeenCalledWith(JSON.stringify(jsonContent));
    expect(provider).toBeDefined();
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', expect.any(Object));
  });

  it('should recursively resolve file:// references in provider config from yaml', async () => {
    const yamlContentWithRefs: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'file://api-key.txt',
        temperature: 'file://temperature.json',
        tools: 'file://tools.yaml',
      },
    };

    const resolvedContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'sk-test-key-12345',
        temperature: 0.7,
        tools: [{ name: 'search', description: 'Search the web' }],
      },
    };

    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContentWithRefs);
    vi.mocked(fileUtil.maybeLoadConfigFromExternalFile).mockReturnValue(resolvedContent);

    const _provider = await loadApiProvider('file://provider.yaml', {
      basePath: '/test',
    });

    expect(fileUtil.maybeLoadConfigFromExternalFile).toHaveBeenCalledWith(yamlContentWithRefs);
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', {
      config: expect.objectContaining({
        apiKey: 'sk-test-key-12345',
        temperature: 0.7,
        tools: [{ name: 'search', description: 'Search the web' }],
      }),
      id: 'openai:chat:gpt-4',
    });
  });

  it('should recursively resolve file:// references in provider config from json', async () => {
    const jsonContentWithRefs: ProviderOptions = {
      id: 'openai:chat:gpt-3.5-turbo',
      config: {
        apiKey: 'file://secrets/api-key.txt',
        systemPrompt: 'file://prompts/system.md',
      },
    };

    const resolvedContent: ProviderOptions = {
      id: 'openai:chat:gpt-3.5-turbo',
      config: {
        apiKey: 'sk-prod-key-67890',
        systemPrompt: 'You are a helpful assistant.',
      },
    };

    vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(jsonContentWithRefs));
    vi.mocked(loadYaml).mockReturnValue(jsonContentWithRefs);
    vi.mocked(fileUtil.maybeLoadConfigFromExternalFile).mockReturnValue(resolvedContent);

    const _provider = await loadApiProvider('file://provider.json', {
      basePath: '/test',
    });

    expect(fileUtil.maybeLoadConfigFromExternalFile).toHaveBeenCalledWith(jsonContentWithRefs);
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-3.5-turbo', {
      config: expect.objectContaining({
        apiKey: 'sk-prod-key-67890',
        systemPrompt: 'You are a helpful assistant.',
      }),
      id: 'openai:chat:gpt-3.5-turbo',
    });
  });

  it('should load Provider from cloud', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://path/to/custom_provider.py:call_api',
      config: {
        apiKey: 'test-key',
        temperature: 0.7,
      },
    });
    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`);

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/custom_provider\.py/),
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: 'test-key',
          temperature: 0.7,
        }),
        id: 'file://path/to/custom_provider.py:call_api',
      }),
    );
  });

  it('should merge local config overrides with cloud provider config', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://providers/custom_llm.py:generate',
      config: {
        apiKey: 'cloud-api-key',
        temperature: 0.7,
        maxTokens: 1000,
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        config: {
          temperature: 0.9, // Override cloud temperature
          topP: 0.95, // Add new config field
        },
      },
    });

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/custom_llm\.py/),
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: 'cloud-api-key', // Preserved from cloud
          temperature: 0.9, // Overridden locally
          maxTokens: 1000, // Preserved from cloud
          topP: 0.95, // Added locally
        }),
        id: 'file://providers/custom_llm.py:generate',
      }),
    );
  });

  it('should override cloud provider label with local label', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://models/sentiment.py:analyze',
      label: 'Cloud Label',
      config: {
        apiKey: 'test-key',
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        label: 'Local Override Label',
      },
    });

    expect(provider).toBeDefined();
    expect(provider.label).toBe('Local Override Label');
  });

  it('should override cloud provider transform with local transform', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://adapters/wrapper.py:call_model',
      transform: 'response.cloudTransform',
      config: {
        apiKey: 'test-key',
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        transform: 'response.localTransform',
      },
    });

    expect(provider).toBeDefined();
    expect(provider.transform).toBe('response.localTransform');
  });

  it('should override cloud provider delay with local delay', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://rate_limited/api.py:fetch',
      delay: 1000,
      config: {
        apiKey: 'test-key',
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        delay: 2000,
      },
    });

    expect(provider).toBeDefined();
    expect(provider.delay).toBe(2000);
  });

  it('should merge cloud provider env with local env overrides', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://integrations/external_api.py:query',
      config: {
        apiKey: 'test-key',
      },
      env: {
        ANTHROPIC_API_KEY: 'cloud-anthropic-key',
        OPENAI_API_KEY: 'cloud-openai-key',
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        env: {
          OPENAI_API_KEY: 'local-openai-key', // Override
          GOOGLE_API_KEY: 'local-google-key', // Add new
        },
      },
    });

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/external_api\.py/),
      expect.objectContaining({
        config: expect.any(Object),
        env: {
          ANTHROPIC_API_KEY: 'cloud-anthropic-key', // Preserved
          OPENAI_API_KEY: 'local-openai-key', // Overridden
          GOOGLE_API_KEY: 'local-google-key', // Added
        },
        id: 'file://integrations/external_api.py:query',
      }),
    );
  });

  it('should merge context env, cloud provider env, and local env overrides', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://integrations/external_api.py:query',
      config: {
        apiKey: 'test-key',
      },
      env: {
        ANTHROPIC_API_KEY: 'cloud-anthropic-key',
        OPENAI_API_KEY: 'cloud-openai-key',
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      env: {
        MISTRAL_API_KEY: 'context-mistral-key',
        OPENAI_API_KEY: 'context-openai-key', // Will be overridden by cloud
      },
      options: {
        env: {
          OPENAI_API_KEY: 'local-openai-key', // Highest priority - overrides both
          GOOGLE_API_KEY: 'local-google-key', // Added by local
        },
      },
    });

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/external_api\.py/),
      expect.objectContaining({
        config: expect.any(Object),
        env: {
          MISTRAL_API_KEY: 'context-mistral-key', // From context
          ANTHROPIC_API_KEY: 'cloud-anthropic-key', // From cloud
          OPENAI_API_KEY: 'local-openai-key', // Local wins (overrides cloud and context)
          GOOGLE_API_KEY: 'local-google-key', // From local
        },
        id: 'file://integrations/external_api.py:query',
      }),
    );
  });

  it('should preserve cloud provider config when no local overrides provided', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://enterprise/secure_llm.py:invoke',
      label: 'Cloud Label',
      transform: 'response.transform',
      delay: 500,
      config: {
        apiKey: 'cloud-key',
        temperature: 0.8,
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`);

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/secure_llm\.py/),
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: 'cloud-key',
          temperature: 0.8,
        }),
        id: 'file://enterprise/secure_llm.py:invoke',
      }),
    );
    expect(provider.transform).toBe('response.transform');
    expect(provider.delay).toBe(500);
  });

  it('should handle cloud provider with empty local config override', async () => {
    vi.mocked(getProviderFromCloud).mockResolvedValue({
      id: 'file://backend/inference.py:predict',
      config: {
        apiKey: 'cloud-key',
        temperature: 0.7,
      },
    });

    const provider = await loadApiProvider(`${CLOUD_PROVIDER_PREFIX}123`, {
      options: {
        config: {}, // Empty override
      },
    });

    expect(provider).toBeDefined();
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/inference\.py/),
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: 'cloud-key',
          temperature: 0.7,
        }),
        id: 'file://backend/inference.py:predict',
      }),
    );
  });

  it('should load OpenAI chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-4.1');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4.1', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5 chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5 chat latest provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5-chat-latest');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-5-chat-latest',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.3 chat latest provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.3-chat-latest');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-5.3-chat-latest',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.4 chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.4');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.4', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.5 chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.5');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.5', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.5 dated snapshot chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.5-2026-04-23');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-5.5-2026-04-23',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should route the new bare gpt-5.6 alias to Chat Completions', async () => {
    const provider = await loadApiProvider('openai:gpt-5.6');

    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.6', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it.each([
    'gpt-5.6-sol',
    'gpt-5.6-terra',
    'gpt-5.6-luna',
  ])('should preserve bare %s Responses routing', async (model) => {
    const provider = await loadApiProvider(`openai:${model}`);

    expect(OpenAiResponsesProvider).toHaveBeenCalledWith(model, expect.any(Object));
    expect(OpenAiChatCompletionProvider).not.toHaveBeenCalled();
    expect(provider).toBeDefined();
  });

  it.each([
    'gpt-5.6',
    'gpt-5.6-sol',
    'gpt-5.6-terra',
    'gpt-5.6-luna',
  ])('should route explicit Chat %s IDs to Chat Completions', async (model) => {
    const provider = await loadApiProvider(`openai:chat:${model}`);

    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(model, expect.any(Object));
    expect(provider).toBeDefined();
  });

  it.each([
    'gpt-5.6',
    'gpt-5.6-sol',
    'gpt-5.6-terra',
    'gpt-5.6-luna',
  ])('should route explicit Responses %s IDs to Responses', async (model) => {
    const provider = await loadApiProvider(`openai:responses:${model}`);

    expect(OpenAiResponsesProvider).toHaveBeenCalledWith(model, expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI Codex provider with model from provider path', async () => {
    const provider = await loadApiProvider('openai:codex:gpt-5.4');

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex:gpt-5.4');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.4');
  });

  it('should load OpenAI Codex provider with GPT-5.5 model from provider path', async () => {
    const provider = await loadApiProvider('openai:codex:gpt-5.5');

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex:gpt-5.5');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.5');
  });

  it('should load OpenAI Codex provider with GPT-5.6 Sol from provider path', async () => {
    const provider = await loadApiProvider('openai:codex:gpt-5.6-sol');

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex:gpt-5.6-sol');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.6-sol');
  });

  it('should load OpenAI Codex SDK provider with model from provider path', async () => {
    const provider = await loadApiProvider('openai:codex-sdk:gpt-5.4-pro');

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex-sdk:gpt-5.4-pro');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.4-pro');
  });

  it('should load OpenAI Codex SDK provider with GPT-5.5 Pro model from provider path', async () => {
    const provider = await loadApiProvider('openai:codex-sdk:gpt-5.5-pro');

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex-sdk:gpt-5.5-pro');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.5-pro');
  });

  it('should load OpenAI Codex provider with model from config when ID has no model', async () => {
    const provider = await loadApiProvider('openai:codex', {
      options: {
        config: {
          model: 'gpt-5.4',
        },
      },
    });

    expect(provider).toBeInstanceOf(OpenAICodexSDKProvider);
    expect(provider.id()).toBe('openai:codex');
    expect((provider as OpenAICodexSDKProvider).config.model).toBe('gpt-5.4');
  });

  it('should load OpenAI GPT-5 nano chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5-nano');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5-nano', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5 mini chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5-mini');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5-mini', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should auto-route OpenAI GPT-5.4 mini shorthand to chat', async () => {
    const originalModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = ['gpt-5.4-mini'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.4-mini');
      expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.4-mini', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalModelNames;
    }
  });

  it('should auto-route OpenAI audio shorthand to chat', async () => {
    const originalChatModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    const originalResponsesModelNames = (OpenAiResponsesProvider as any)
      .OPENAI_RESPONSES_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = ['gpt-audio'];
    (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = [];
    try {
      const provider = await loadApiProvider('openai:gpt-audio');
      expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-audio', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalChatModelNames;
      (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = originalResponsesModelNames;
    }
  });

  it('should default OpenAI realtime to a current model', async () => {
    const provider = await loadApiProvider('openai:realtime');
    expect(OpenAiRealtimeProvider).toHaveBeenCalledWith('gpt-realtime-1.5', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should auto-route OpenAI realtime 2 shorthand to realtime', async () => {
    const originalModelNames = (OpenAiRealtimeProvider as any).OPENAI_REALTIME_MODEL_NAMES;
    (OpenAiRealtimeProvider as any).OPENAI_REALTIME_MODEL_NAMES = ['gpt-realtime-2'];
    try {
      const provider = await loadApiProvider('openai:gpt-realtime-2');
      expect(OpenAiRealtimeProvider).toHaveBeenCalledWith('gpt-realtime-2', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiRealtimeProvider as any).OPENAI_REALTIME_MODEL_NAMES = originalModelNames;
    }
  });

  it('should auto-route OpenAI GPT-5.4 nano shorthand to chat', async () => {
    const originalModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = ['gpt-5.4-nano'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.4-nano');
      expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.4-nano', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalModelNames;
    }
  });

  it('should auto-route OpenAI GPT-5.4 mini dated snapshot shorthand to chat', async () => {
    const originalModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = ['gpt-5.4-mini-2026-03-17'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.4-mini-2026-03-17');
      expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
        'gpt-5.4-mini-2026-03-17',
        expect.any(Object),
      );
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalModelNames;
    }
  });

  it('should auto-route OpenAI GPT-5.4 nano dated snapshot shorthand to chat', async () => {
    const originalModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = ['gpt-5.4-nano-2026-03-17'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.4-nano-2026-03-17');
      expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
        'gpt-5.4-nano-2026-03-17',
        expect.any(Object),
      );
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalModelNames;
    }
  });

  it('should auto-route OpenAI GPT-5.4 Pro shorthand to responses', async () => {
    const originalChatModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    const originalResponsesModelNames = (OpenAiResponsesProvider as any)
      .OPENAI_RESPONSES_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = [];
    (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = ['gpt-5.4-pro'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.4-pro');
      expect(OpenAiResponsesProvider).toHaveBeenCalledWith('gpt-5.4-pro', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalChatModelNames;
      (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = originalResponsesModelNames;
    }
  });

  it('should auto-route OpenAI GPT-5.5 Pro shorthand to responses', async () => {
    const originalChatModelNames = (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES;
    const originalResponsesModelNames = (OpenAiResponsesProvider as any)
      .OPENAI_RESPONSES_MODEL_NAMES;
    (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = [];
    (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = ['gpt-5.5-pro'];
    try {
      const provider = await loadApiProvider('openai:gpt-5.5-pro');
      expect(OpenAiResponsesProvider).toHaveBeenCalledWith('gpt-5.5-pro', expect.any(Object));
      expect(provider).toBeDefined();
    } finally {
      (OpenAiChatCompletionProvider as any).OPENAI_CHAT_MODEL_NAMES = originalChatModelNames;
      (OpenAiResponsesProvider as any).OPENAI_RESPONSES_MODEL_NAMES = originalResponsesModelNames;
    }
  });

  it('should load OpenAI GPT-5.4 nano chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.4-nano');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.4-nano', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.4 nano dated snapshot chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.4-nano-2026-03-17');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-5.4-nano-2026-03-17',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load OpenAI GPT-5.4 mini chat provider', async () => {
    const provider = await loadApiProvider('openai:chat:gpt-5.4-mini');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-5.4-mini', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI chat provider with default model', async () => {
    const provider = await loadApiProvider('openai:chat');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-4.1-2025-04-14',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load OpenAI chat provider with model from config when ID has no model', async () => {
    const provider = await loadApiProvider('openai:chat', {
      options: {
        config: {
          model: 'DeepSeek-R1',
        },
      },
    });
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('DeepSeek-R1', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load OpenAI embedding provider', async () => {
    const provider = await loadApiProvider('openai:embedding');
    expect(OpenAiEmbeddingProvider).toHaveBeenCalledWith(
      'text-embedding-3-large',
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load DeepSeek provider with default model', async () => {
    const provider = await loadApiProvider('deepseek:');
    expect(provider).toBeDefined();
  });

  it('should load DeepSeek provider with specific model', async () => {
    const provider = await loadApiProvider('deepseek:deepseek-reasoner');
    expect(provider).toBeDefined();
  });

  it('should load Hyperbolic provider with specific model', async () => {
    const provider = await loadApiProvider('hyperbolic:meta-llama/Meta-Llama-3-8B-Instruct-Turbo');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'meta-llama/Meta-Llama-3-8B-Instruct-Turbo',
      {
        config: expect.objectContaining({
          apiBaseUrl: 'https://api.hyperbolic.xyz/v1',
          apiKeyEnvar: 'HYPERBOLIC_API_KEY',
        }),
      },
    );
    expect(provider).toBeDefined();
  });

  it('should load GitHub provider with default model', async () => {
    const provider = await loadApiProvider('github:');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-5', {
      config: expect.objectContaining({
        apiBaseUrl: 'https://models.github.ai/inference',
        apiKeyEnvar: 'GITHUB_TOKEN',
      }),
    });
    expect(provider).toBeDefined();
  });

  it('should load GitHub provider with specific model', async () => {
    const provider = await loadApiProvider('github:openai/gpt-4o-mini');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-4o-mini', {
      config: expect.objectContaining({
        apiBaseUrl: 'https://models.github.ai/inference',
        apiKeyEnvar: 'GITHUB_TOKEN',
      }),
    });
    expect(provider).toBeDefined();
  });

  it('should load HTTP provider', async () => {
    const provider = await loadApiProvider('http://test.com');
    expect(HttpProvider).toHaveBeenCalledWith('http://test.com', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load HTTPS provider', async () => {
    const provider = await loadApiProvider('https://test.com');
    expect(HttpProvider).toHaveBeenCalledWith('https://test.com', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load WebSocket provider', async () => {
    const provider = await loadApiProvider('ws://test.com');
    expect(WebSocketProvider).toHaveBeenCalledWith('ws://test.com', expect.any(Object));
    expect(provider).toBeDefined();
  });

  it('should load script provider', async () => {
    const provider = await loadApiProvider('exec:test.sh');
    expect(ScriptCompletionProvider).toHaveBeenCalledWith(
      expect.stringMatching(/test\.sh$/),
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load Python provider', async () => {
    const provider = await loadApiProvider('python:test.py');
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/test\.py$/),
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should load Python provider from file path', async () => {
    const provider = await loadApiProvider('file://test.py');
    expect(PythonProvider).toHaveBeenCalledWith(
      expect.stringMatching(/test\.py$/),
      expect.any(Object),
    );
    expect(provider).toBeDefined();
  });

  it('should handle unidentified provider', async () => {
    await expect(loadApiProvider('unknown:provider')).rejects.toThrow(
      'Could not identify provider',
    );
  });

  it('should load JFrog ML provider', async () => {
    const provider = await loadApiProvider('jfrog:test-model');
    expect(provider).toBeDefined();
  });

  it('should handle invalid file path for yaml/json config', async () => {
    vi.mocked(fs.readFileSync).mockImplementation(() => {
      throw new Error('File not found');
    });
    await expect(loadApiProvider('file://invalid.yaml')).rejects.toThrow('File not found');
  });

  it('should handle invalid yaml content', async () => {
    vi.mocked(fs.readFileSync).mockReturnValue('invalid: yaml: content:');
    vi.mocked(loadYaml).mockReturnValue(null);
    await expect(loadApiProvider('file://invalid.yaml')).rejects.toThrow('Provider config');
  });

  it('should handle yaml config without id', async () => {
    vi.mocked(fs.readFileSync).mockReturnValue('config:\n  key: value');
    vi.mocked(loadYaml).mockReturnValue({ config: { key: 'value' } });
    await expect(loadApiProvider('file://invalid.yaml')).rejects.toThrow('must have an id');
  });

  it('should handle provider with custom base path', async () => {
    const mockProvider = {
      id: () => 'python:script.py',
      config: {
        basePath: '/custom/path',
      },
      callApi: async (input: string) => ({ output: input }),
    };
    vi.mocked(PythonProvider).mockImplementation(function (this: any) {
      Object.assign(this, mockProvider);
      return this;
    } as any);

    const provider = await loadApiProvider('python:script.py', {
      basePath: '/custom/path',
      options: {
        config: {},
      },
    });
    expect(provider.config.basePath).toBe('/custom/path');
  });

  it('should handle provider with delay', async () => {
    const provider = await loadApiProvider('echo', {
      options: {
        delay: 1000,
      },
    });
    expect(provider.delay).toBe(1000);
  });

  it('should handle provider with custom label template', async () => {
    mockProcessEnv({ CUSTOM_LABEL: 'my-label' });
    const provider = await loadApiProvider('echo', {
      options: {
        label: '{{ env.CUSTOM_LABEL }}',
      },
    });
    expect(provider.label).toBe('my-label');
    mockProcessEnv({ CUSTOM_LABEL: undefined });
  });

  it('should throw error when file provider array is loaded with loadApiProvider', async () => {
    const yamlContent = [
      {
        id: 'openai:chat:gpt-4',
        config: { apiKey: 'test-key1' },
      },
      {
        id: 'anthropic:claude-2',
        config: { apiKey: 'test-key2' },
      },
    ];
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    await expect(loadApiProvider('file://test.yaml')).rejects.toThrow(
      'Multiple providers found in test.yaml. Use loadApiProviders instead of loadApiProvider.',
    );
  });

  it('should handle file provider with environment variables', async () => {
    mockProcessEnv({ OPENAI_API_KEY: 'test-key-from-env' });
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: '{{ env.OPENAI_API_KEY }}',
      },
      env: {
        OPENAI_API_KEY: 'override-key',
      },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const provider = await loadApiProvider('file://test.yaml', {
      basePath: '/test',
      env: { OPENAI_API_KEY: 'final-override-key' },
    });

    expect(provider).toBeDefined();
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-4',
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: expect.any(String),
        }),
        env: expect.objectContaining({
          OPENAI_API_KEY: 'final-override-key',
        }),
      }),
    );
    mockProcessEnv({ OPENAI_API_KEY: undefined });
  });

  it('should load multiple providers from yaml file using loadApiProviders', async () => {
    const yamlContent = [
      {
        id: 'openai:chat:gpt-4o-mini',
        config: { apiKey: 'test-key1' },
      },
      {
        id: 'anthropic:claude-3-7-sonnet-20250219',
        config: { apiKey: 'test-key2' },
      },
    ];
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const providers = await loadApiProviders('file://test.yaml');
    expect(providers).toHaveLength(2);
    expect(providers[0]).toBeDefined();
    expect(providers[1]).toBeDefined();
    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('test.yaml'), 'utf8');
    expect(loadYaml).toHaveBeenCalledWith('yaml content');
  });

  it('should handle absolute file paths', async () => {
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: { apiKey: 'test-key' },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const absolutePath = path.resolve('/absolute/path/to/providers.yaml');
    const provider = await loadApiProvider(`file://${absolutePath}`);

    expect(provider).toBeDefined();
    expect(fs.readFileSync).toHaveBeenCalledWith(absolutePath, 'utf8');
  });

  it('should handle provider with null or undefined config values', async () => {
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'test-key',
        nullValue: null,
        undefinedValue: undefined,
      },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const provider = await loadApiProvider('file://test.yaml');
    expect(provider).toBeDefined();
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-4',
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: 'test-key',
        }),
      }),
    );
  });

  it('should handle provider with undefined options', async () => {
    const provider = await loadApiProvider('echo', {
      options: undefined,
    });
    expect(provider).toBeDefined();
  });

  it('should throw error for invalid providerPaths type', async () => {
    // Test with a number, which is an invalid type
    await expect(loadApiProviders(42 as any)).rejects.toThrow('Invalid providers list');

    // Test with an object that doesn't match any valid format
    await expect(loadApiProviders({ foo: 'bar' } as any)).rejects.toThrow('Invalid providers list');
  });

  it('should handle non-yaml/json file paths', async () => {
    // Test with a text file path
    await expect(loadApiProviders('file://test.txt')).rejects.toThrow(
      /Could not identify provider/,
    );
  });
});

describe('loadApiProviders', () => {
  beforeEach(() => {
    vi.resetAllMocks();
    cliState.config = undefined;

    // Reset maybeLoadConfigFromExternalFile mock to default implementation
    vi.mocked(fileUtil.maybeLoadConfigFromExternalFile).mockImplementation((input: any) => input);
  });

  it('should load single provider from string', async () => {
    const providers = await loadApiProviders('echo');
    expect(providers).toHaveLength(1);
    expect(providers[0].id()).toBe('echo');
  });

  it('should load multiple providers from array of strings', async () => {
    const providers = await loadApiProviders(['echo', 'openai:chat:gpt-4']);
    expect(providers).toHaveLength(2);
    expect(providers[0].id()).toBe('echo');
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', expect.any(Object));
  });

  it('should load provider from function', async () => {
    const customFunction = async (prompt: string) => ({ output: prompt });
    const providers = await loadApiProviders(customFunction);
    expect(providers).toHaveLength(1);
    expect(providers[0].id()).toBe('custom-function');
    await expect(providers[0].callApi('test')).resolves.toEqual({ output: 'test' });
  });

  it('should load provider from function with label', async () => {
    const customFunction = async (prompt: string) => ({ output: prompt });
    customFunction.label = 'custom-label';
    const providers = await loadApiProviders([customFunction]);
    expect(providers).toHaveLength(1);
    expect(providers[0].id()).toBe('custom-label');
  });

  it('should load provider from options object', async () => {
    const options: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: 'test-key',
      },
    };
    const providers = await loadApiProviders([options]);
    expect(providers).toHaveLength(1);
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', expect.any(Object));
  });

  it('should load provider from options map', async () => {
    const providers = await loadApiProviders([
      {
        'openai:chat:gpt-4': {
          config: {
            apiKey: 'test-key',
          },
        },
      },
    ]);
    expect(providers).toHaveLength(1);
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('gpt-4', expect.any(Object));
  });

  it('should throw error for invalid providers list', async () => {
    await expect(loadApiProviders({} as any)).rejects.toThrow('Invalid providers list');
  });

  it('should handle loadApiProviders with empty array', async () => {
    const providers = await loadApiProviders([]);
    expect(providers).toHaveLength(0);
  });

  it('should handle loadApiProviders with mixed provider types', async () => {
    const customFunction = async (prompt: string) => ({ output: prompt });
    const providers = await loadApiProviders([
      'echo',
      customFunction,
      { id: 'openai:chat', config: {} },
      { 'openai:completion': { config: {} } },
    ]);
    expect(providers).toHaveLength(4);
  });

  it('should handle provider with null config', async () => {
    const provider = await loadApiProvider('echo', {
      options: {
        config: null,
      },
    });
    expect(provider).toBeDefined();
  });

  it('should handle provider with undefined options', async () => {
    const provider = await loadApiProvider('echo', {
      options: undefined,
    });
    expect(provider).toBeDefined();
  });

  it('should handle relative file paths', async () => {
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: { apiKey: 'test-key' },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const relativePath = 'relative/path/to/providers.yaml';
    const providers = await loadApiProviders(`file://${relativePath}`, {
      basePath: '/test/base/path',
    });

    expect(providers).toHaveLength(1);
    expect(fs.readFileSync).toHaveBeenCalledWith(
      path.join('/test/base/path', relativePath),
      'utf8',
    );
  });

  it('should handle absolute file paths in loadApiProviders', async () => {
    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: { apiKey: 'test-key' },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const absolutePath = path.resolve('/absolute/path/to/providers.yaml');
    const providers = await loadApiProviders(`file://${absolutePath}`);

    expect(providers).toHaveLength(1);
    expect(fs.readFileSync).toHaveBeenCalledWith(absolutePath, 'utf8');
    expect(loadYaml).toHaveBeenCalledWith('yaml content');
  });

  it('should load multiple providers from a file specified in a providers array', async () => {
    // Setup mock file with multiple providers
    const yamlContent = [
      {
        id: 'echo',
        config: { prefix: 'Echo Provider: ' },
      },
      {
        id: 'openai:gpt-4o-mini',
        config: { temperature: 0.1 },
      },
    ];
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    // Create provider array with a mix of direct provider and file reference
    const providerArray = [
      'anthropic:claude-3-5-sonnet-20241022',
      'file://./providers.yaml', // This should expand to the two providers above
    ];

    const providers = await loadApiProviders(providerArray);

    // We should get 3 providers: 1 direct + 2 from file
    expect(providers).toHaveLength(3);

    // Just verify that all providers are defined
    providers.forEach((provider) => {
      expect(provider).toBeDefined();
    });

    // Verify file was read correctly
    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('providers.yaml'), 'utf8');
    expect(loadYaml).toHaveBeenCalledWith('yaml content');
  });

  it('should handle nested arrays of providers from multiple file references', async () => {
    // First file
    const firstFileContent = [
      {
        id: 'echo',
        config: { prefix: 'First file: ' },
      },
    ];

    // Second file
    const secondFileContent = [
      {
        id: 'openai:gpt-4o-mini',
        config: { temperature: 0.1 },
      },
      {
        id: 'anthropic:claude-3-5-sonnet-20241022',
        config: { temperature: 0.7 },
      },
    ];

    // Mock the file system read for different paths
    vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
      if (filePath.toString().includes('first.yaml')) {
        return 'first file content';
      } else if (filePath.toString().includes('second.yaml')) {
        return 'second file content';
      }
      return '';
    });

    // Mock yaml loading based on different file contents
    vi.mocked(loadYaml).mockImplementation((content) => {
      if (content === 'first file content') {
        return firstFileContent;
      } else if (content === 'second file content') {
        return secondFileContent;
      }
      return null;
    });

    // Provider array with multiple file references
    const providerArray = ['file://./first.yaml', 'file://./second.yaml'];

    const providers = await loadApiProviders(providerArray);

    // We should get 3 providers total: 1 from first file + 2 from second file
    expect(providers).toHaveLength(3);

    // Just verify all providers are defined
    providers.forEach((provider) => {
      expect(provider).toBeDefined();
    });

    // Verify both files were read
    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('first.yaml'), 'utf8');
    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('second.yaml'), 'utf8');
  });

  it('should use env values from cliState.config', async () => {
    // Set up dummy config with env block
    cliState.config = {
      env: {
        TEST_API_KEY: 'test-key-from-cli-state',
        OTHER_VAR: 'other-value',
      },
    };

    const yamlContent: ProviderOptions = {
      id: 'openai:chat:gpt-4',
      config: {
        apiKey: '{{ env.TEST_API_KEY }}',
      },
    };
    vi.mocked(fs.readFileSync).mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockReturnValue(yamlContent);

    const providers = await loadApiProviders('file://test.yaml');

    expect(providers).toHaveLength(1);
    expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith(
      'gpt-4',
      expect.objectContaining({
        config: expect.objectContaining({
          apiKey: expect.any(String),
        }),
        env: expect.objectContaining({
          TEST_API_KEY: 'test-key-from-cli-state',
          OTHER_VAR: 'other-value',
        }),
      }),
    );
  });
});