promptfoo--promptfoo/test/providers/mistral.test.ts

import { beforeEach, describe, expect, it, vi } from 'vitest';
import { fetchWithCache, getCache, isCacheEnabled, withCacheNamespace } from '../../src/cache';
import logger from '../../src/logger';
import {
  MistralChatCompletionProvider,
  MistralEmbeddingProvider,
} from '../../src/providers/mistral';
import { maybeLoadToolsFromExternalFile } from '../../src/util';

vi.mock('../../src/logger', () => ({
  default: {
    debug: vi.fn(),
    error: vi.fn(),
    info: vi.fn(),
    warn: vi.fn(),
  },
}));

vi.mock('../../src/cache', async () => ({
  ...(await vi.importActual('../../src/cache')),
  fetchWithCache: vi.fn(),
  getCache: vi.fn(),
  isCacheEnabled: vi.fn(),
}));

vi.mock('../../src/util', async () => ({
  ...(await vi.importActual('../../src/util')),
  maybeLoadToolsFromExternalFile: vi.fn(),
}));

describe('Mistral', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    vi.mocked(fetchWithCache).mockReset();
    vi.mocked(maybeLoadToolsFromExternalFile).mockImplementation(async (tools) => tools);
    vi.mocked(isCacheEnabled).mockReturnValue(false);
    vi.mocked(getCache).mockReturnValue({
      get: vi.fn().mockResolvedValue(null),
      set: vi.fn(),
      wrap: vi.fn(),
      del: vi.fn(),
      clear: vi.fn(),
      stores: [
        {
          get: vi.fn(),
          set: vi.fn(),
        },
      ] as any,
      mget: vi.fn(),
      mset: vi.fn(),
      mdel: vi.fn(),
      reset: vi.fn(),
      ttl: vi.fn(),
      on: vi.fn(),
      removeAllListeners: vi.fn(),
    } as any);
  });

  describe('MistralChatCompletionProvider', () => {
    let provider: MistralChatCompletionProvider;

    beforeEach(() => {
      provider = new MistralChatCompletionProvider('mistral-tiny');
      vi.spyOn(provider, 'getApiKey').mockReturnValue('fake-api-key');
    });

    it('should create a provider with default options', () => {
      expect(provider.modelName).toBe('mistral-tiny');
      expect(provider.config).toEqual({});
    });

    it('should create a provider with custom options', () => {
      const customProvider = new MistralChatCompletionProvider('mistral-medium', {
        config: { temperature: 0.7 },
      });
      expect(customProvider.modelName).toBe('mistral-medium');
      expect(customProvider.config).toEqual({ temperature: 0.7 });
    });

    it('should support current Mistral model families', () => {
      const smallProvider = new MistralChatCompletionProvider('magistral-small-2509');
      expect(smallProvider.modelName).toBe('magistral-small-2509');
      expect(smallProvider.config).toEqual({});

      const magistralMediumProvider = new MistralChatCompletionProvider('magistral-medium-latest', {
        config: { temperature: 0.7, max_tokens: 40960 },
      });
      expect(magistralMediumProvider.modelName).toBe('magistral-medium-latest');
      expect(magistralMediumProvider.config).toEqual({ temperature: 0.7, max_tokens: 40960 });

      expect(new MistralChatCompletionProvider('mistral-large-2512').modelName).toBe(
        'mistral-large-2512',
      );
      expect(new MistralChatCompletionProvider('mistral-medium-3.5').modelName).toBe(
        'mistral-medium-3.5',
      );
      expect(new MistralChatCompletionProvider('ministral-14b-latest').modelName).toBe(
        'ministral-14b-latest',
      );
    });

    it('should support Pixtral multimodal model', () => {
      const pixtralProvider = new MistralChatCompletionProvider('pixtral-12b', {
        config: { temperature: 0.8, max_tokens: 2048 },
      });
      expect(pixtralProvider.modelName).toBe('pixtral-12b');
      expect(pixtralProvider.config).toEqual({ temperature: 0.8, max_tokens: 2048 });
    });

    it('should calculate cost correctly for Pixtral model', async () => {
      const pixtralProvider = new MistralChatCompletionProvider('pixtral-12b');
      vi.spyOn(pixtralProvider, 'getApiKey').mockReturnValue('fake-api-key');

      const mockResponse = {
        choices: [{ message: { content: 'Image analysis response' } }],
        usage: { total_tokens: 2000, prompt_tokens: 800, completion_tokens: 1200 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await pixtralProvider.callApi('Analyze this image: <image_url>');

      // Pixtral: $0.15/M tokens for both input and output
      // 800 prompt tokens * 0.15/1M + 1200 completion tokens * 0.15/1M = 0.00012 + 0.00018 = 0.0003
      expect(result.cost).toBeCloseTo(0.0003, 6);
    });

    it('should call Mistral API and return output with correct structure', async () => {
      const mockResponse = {
        choices: [{ message: { content: 'Test output' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        expect.stringContaining('/chat/completions'),
        expect.objectContaining({
          method: 'POST',
          headers: expect.objectContaining({
            'Content-Type': 'application/json',
            'x-promptfoo-silent': 'true',
            Authorization: expect.stringContaining('Bearer '),
          }),
          body: expect.stringContaining('"messages":[{"role":"user","content":"Test prompt"}]'),
        }),
        expect.any(Number),
        'json',
        true,
      );

      expect(result).toEqual({
        output: 'Test output',
        tokenUsage: {
          total: 10,
          prompt: 5,
          completion: 5,
          numRequests: 1,
        },
        cached: false,
        cost: expect.any(Number),
      });
    });

    it('should preserve explicit zero for top_p, random_seed, and max_tokens', async () => {
      const zeroProvider = new MistralChatCompletionProvider('mistral-tiny', {
        config: { top_p: 0, random_seed: 0, max_tokens: 0 },
      });
      vi.spyOn(zeroProvider, 'getApiKey').mockReturnValue('fake-api-key');

      const mockResponse = {
        choices: [{ message: { content: 'Test output' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await zeroProvider.callApi('Test prompt');

      const callArgs = vi.mocked(fetchWithCache).mock.calls[0];
      const body = JSON.parse((callArgs[1] as RequestInit).body as string);
      expect(body.top_p).toBe(0);
      expect(body.random_seed).toBe(0);
      expect(body.max_tokens).toBe(0);
    });

    it('should pass through current chat completion options', async () => {
      const advancedProvider = new MistralChatCompletionProvider('mistral-medium-3.5', {
        config: {
          frequency_penalty: 0.25,
          presence_penalty: 0.5,
          stop: ['END'],
          n: 2,
          safe_prompt: false,
          prompt_mode: null,
          reasoning_effort: 'high',
          prediction: { type: 'content', content: 'Expected completion' },
          metadata: { requestId: 'abc-123' },
          guardrails: [{ block_on_error: true }],
          response_format: {
            type: 'json_schema',
            json_schema: {
              name: 'answer',
              schema: {
                type: 'object',
                properties: {
                  answer: { type: 'string' },
                },
                required: ['answer'],
              },
            },
          },
        },
      });
      vi.spyOn(advancedProvider, 'getApiKey').mockReturnValue('fake-api-key');
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: '{"answer":"ok"}' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await advancedProvider.callApi('Test prompt');

      const requestInit = vi.mocked(fetchWithCache).mock.calls[0]?.[1];
      expect(requestInit).toBeDefined();
      const body = JSON.parse((requestInit as RequestInit).body as string);
      expect(body).toMatchObject({
        frequency_penalty: 0.25,
        presence_penalty: 0.5,
        stop: ['END'],
        n: 2,
        safe_prompt: false,
        prompt_mode: null,
        reasoning_effort: 'high',
        prediction: { type: 'content', content: 'Expected completion' },
        metadata: { requestId: 'abc-123' },
        guardrails: [{ block_on_error: true }],
        response_format: {
          type: 'json_schema',
          json_schema: {
            name: 'answer',
          },
        },
      });
    });

    it('should include tools configuration in the request body', async () => {
      const tools = [
        {
          type: 'function',
          function: {
            name: 'calculate',
            description: 'Perform basic math',
            parameters: {
              type: 'object',
              properties: {
                a: { type: 'number' },
                b: { type: 'number' },
              },
              required: ['a', 'b'],
            },
          },
        },
      ];
      const customProvider = new MistralChatCompletionProvider('mistral-medium', {
        config: {
          tools,
          tool_choice: 'any',
          parallel_tool_calls: false,
        },
      });
      vi.spyOn(customProvider, 'getApiKey').mockReturnValue('fake-api-key');
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: null, tool_calls: tools } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await customProvider.callApi('Test prompt');

      expect(vi.mocked(maybeLoadToolsFromExternalFile)).toHaveBeenCalledWith(tools, undefined);
      const requestInit = vi.mocked(fetchWithCache).mock.calls[0]?.[1];
      expect(requestInit).toBeDefined();
      const requestBody = JSON.parse((requestInit as RequestInit).body as string);
      expect(requestBody).toMatchObject({
        tools,
        tool_choice: 'any',
        parallel_tool_calls: false,
      });
    });

    it('should calculate cost correctly for Magistral models', async () => {
      const magistralSmallProvider = new MistralChatCompletionProvider('magistral-small-2509');
      vi.spyOn(magistralSmallProvider, 'getApiKey').mockReturnValue('fake-api-key');

      const mockResponse = {
        choices: [{ message: { content: 'Reasoning response' } }],
        usage: { total_tokens: 1000, prompt_tokens: 100, completion_tokens: 900 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await magistralSmallProvider.callApi('Test reasoning prompt');

      // Magistral Small: $0.5/M input, $1.5/M output
      // 100 prompt tokens * 0.5/1M + 900 completion tokens * 1.5/1M = 0.00005 + 0.00135 = 0.0014
      expect(result.cost).toBeCloseTo(0.0014, 6);
    });

    it('should calculate cost correctly for current latest aliases', async () => {
      const latestProvider = new MistralChatCompletionProvider('mistral-large-latest');
      vi.spyOn(latestProvider, 'getApiKey').mockReturnValue('fake-api-key');

      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: 'Current latest response' } }],
          usage: { total_tokens: 1000, prompt_tokens: 400, completion_tokens: 600 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await latestProvider.callApi('Test latest alias pricing');

      // mistral-large-latest currently tracks mistral-large-2512 pricing: $0.5/M in, $1.5/M out
      expect(result.cost).toBeCloseTo(0.0011, 6);
    });

    // Regression coverage: Mistral silently repoints `*-latest`/bare aliases to newer
    // models. These lock the hardcoded pricing to whatever the alias resolves to today.
    it.each([
      // [model, input price/M, output price/M, expected cost for 400 in / 600 out]
      // mistral-small-latest -> Mistral Small 4 (mistral-small-2603): $0.15/$0.60
      ['mistral-small-latest', 0.00042],
      // magistral-small-latest folded into Mistral Small 4: $0.15/$0.60
      ['magistral-small-latest', 0.00042],
      // mistral-medium-latest + bare mistral-medium -> Mistral Medium 3.5: $1.50/$7.50
      ['mistral-medium-latest', 0.0051],
      ['mistral-medium', 0.0051],
      // mistral-medium-2604 is the canonical dated ID for Mistral Medium 3.5
      ['mistral-medium-2604', 0.0051],
      // version aliases that also resolve to Mistral Medium 3.5
      ['mistral-medium-3-5', 0.0051],
      // Mistral Code product aliases resolve to Codestral: $0.30/$0.90
      ['mistral-code-latest', 0.00066],
      // Devstral 2 agent alias: $0.40/$2.00
      ['mistral-code-agent-latest', 0.00136],
    ])('tracks current pricing for %s', async (model, expectedCost) => {
      const provider = new MistralChatCompletionProvider(model);
      vi.spyOn(provider, 'getApiKey').mockReturnValue('fake-api-key');

      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: 'ok' } }],
          usage: { total_tokens: 1000, prompt_tokens: 400, completion_tokens: 600 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test alias pricing');
      expect(result.cost).toBeCloseTo(expectedCost, 6);
    });

    it('forwards prompt_cache_key in the request body', async () => {
      const provider = new MistralChatCompletionProvider('mistral-large-latest', {
        config: { prompt_cache_key: 'shared-prefix-1' },
      });
      vi.spyOn(provider, 'getApiKey').mockReturnValue('fake-api-key');

      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: 'ok' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await provider.callApi('Test prompt cache key');

      const requestInit = vi.mocked(fetchWithCache).mock.calls[0]?.[1];
      const body = JSON.parse(requestInit?.body as string);
      expect(body.prompt_cache_key).toBe('shared-prefix-1');
    });

    it('should use cache when enabled', async () => {
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: vi.fn().mockResolvedValue({
          output: 'Cached output',
          tokenUsage: { total: 10, prompt: 5, completion: 5 },
          cost: 0.000005,
        }),
        set: vi.fn(),
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);
      const result = await provider.callApi('Test prompt');

      expect(result.cached).toBe(true);
      expect(result).toEqual({
        output: 'Cached output',
        tokenUsage: {
          total: 10,
          prompt: 5,
          completion: 5,
          cached: 10,
        },
        cached: true,
        cost: 0.000005,
      });
      expect(vi.mocked(fetchWithCache)).not.toHaveBeenCalled();
    });

    it('should hash request params in cache keys', async () => {
      const cacheGet = vi.fn().mockResolvedValue(null);
      const cacheSet = vi.fn();
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: cacheGet,
        set: cacheSet,
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: 'Fresh output' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await provider.callApi('Sensitive prompt sk-mistral-secret');

      const cacheKey = cacheGet.mock.calls[0]?.[0] as string;
      expect(cacheKey).toMatch(
        /^mistral:chat:mistral-tiny:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
      );
      expect(cacheKey).not.toContain('Sensitive prompt');
      expect(cacheKey).not.toContain('sk-mistral-secret');
      expect(cacheSet).toHaveBeenCalledWith(
        cacheKey,
        expect.objectContaining({ output: 'Fresh output' }),
      );
    });

    it('should isolate hashed cache keys by resolved API key', async () => {
      const cacheGet = vi.fn().mockResolvedValue(null);
      const cacheSet = vi.fn();
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: cacheGet,
        set: cacheSet,
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);
      const providerA = new MistralChatCompletionProvider('mistral-tiny');
      const providerB = new MistralChatCompletionProvider('mistral-tiny');
      vi.spyOn(providerA, 'getApiKey').mockReturnValue('sk-mistral-tenant-a');
      vi.spyOn(providerB, 'getApiKey').mockReturnValue('sk-mistral-tenant-b');
      vi.spyOn(providerA, 'getApiUrl').mockReturnValue('https://shared.mistral.example/v1');
      vi.spyOn(providerB, 'getApiUrl').mockReturnValue('https://shared.mistral.example/v1');
      vi.mocked(fetchWithCache)
        .mockResolvedValueOnce({
          data: {
            choices: [{ message: { content: 'Tenant A output' } }],
            usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
          },
          cached: false,
          status: 200,
          statusText: 'OK',
        })
        .mockResolvedValueOnce({
          data: {
            choices: [{ message: { content: 'Tenant B output' } }],
            usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
          },
          cached: false,
          status: 200,
          statusText: 'OK',
        });

      await providerA.callApi('Shared sensitive prompt');
      await providerB.callApi('Shared sensitive prompt');

      const cacheKeyA = cacheGet.mock.calls[0][0] as string;
      const cacheKeyB = cacheGet.mock.calls[1][0] as string;
      expect(cacheKeyA).toMatch(
        /^mistral:chat:mistral-tiny:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
      );
      expect(cacheKeyB).toMatch(
        /^mistral:chat:mistral-tiny:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
      );
      expect(cacheKeyA).not.toBe(cacheKeyB);
      expect(fetchWithCache).toHaveBeenCalledTimes(2);
      for (const cacheKey of [cacheKeyA, cacheKeyB]) {
        expect(cacheKey).not.toContain('Shared sensitive prompt');
        expect(cacheKey).not.toContain('sk-mistral-tenant-a');
        expect(cacheKey).not.toContain('sk-mistral-tenant-b');
        expect(cacheKey).not.toContain('shared.mistral.example');
      }
    });

    it('should keep auth cache identity stable across module reloads', async () => {
      async function getCacheKeyFromFreshModule() {
        vi.resetModules();
        const freshCache = await import('../../src/cache');
        const cacheGet = vi.fn().mockResolvedValue({
          output: 'Cached output',
          tokenUsage: { total: 10, prompt: 5, completion: 5 },
        });
        vi.mocked(freshCache.isCacheEnabled).mockReturnValue(true);
        vi.mocked(freshCache.getCache).mockReturnValue({
          get: cacheGet,
          set: vi.fn(),
        } as any);
        const { MistralChatCompletionProvider: FreshMistralChatCompletionProvider } = await import(
          '../../src/providers/mistral'
        );
        const freshProvider = new FreshMistralChatCompletionProvider('mistral-tiny', {
          config: {
            apiKey: 'sk-mistral-reload-secret',
            apiBaseUrl: 'https://shared.mistral.example/v1',
          },
        });

        await freshProvider.callApi('Shared sensitive prompt');
        return cacheGet.mock.calls[0][0] as string;
      }

      const firstCacheKey = await getCacheKeyFromFreshModule();
      const secondCacheKey = await getCacheKeyFromFreshModule();

      expect(firstCacheKey).toBe(secondCacheKey);
      expect(firstCacheKey).toMatch(
        /^mistral:chat:mistral-tiny:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
      );
      expect(firstCacheKey).not.toContain('sk-mistral-reload-secret');
      expect(firstCacheKey).not.toContain('Shared sensitive prompt');
      expect(firstCacheKey).not.toContain('shared.mistral.example');
    });

    it('should deduplicate in-flight chat requests without using raw fetch cache keys', async () => {
      const mockResponse = {
        choices: [{ message: { content: 'Shared output' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      let resolveFetch: (value: any) => void;
      vi.mocked(fetchWithCache).mockReturnValueOnce(
        new Promise<any>((resolve) => {
          resolveFetch = resolve;
        }) as ReturnType<typeof fetchWithCache>,
      );

      const first = provider.callApi('Concurrent sensitive prompt');
      const second = provider.callApi('Concurrent sensitive prompt');

      await Promise.resolve();
      expect(fetchWithCache).toHaveBeenCalledTimes(1);

      resolveFetch!({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await expect(Promise.all([first, second])).resolves.toEqual([
        expect.objectContaining({ output: 'Shared output' }),
        expect.objectContaining({ output: 'Shared output' }),
      ]);
    });

    it('should isolate in-flight chat request dedupe by cache namespace', async () => {
      const resolveFetches: Array<(value: any) => void> = [];
      vi.mocked(fetchWithCache)
        .mockReturnValueOnce(
          new Promise<any>((resolve) => {
            resolveFetches.push(resolve);
          }) as ReturnType<typeof fetchWithCache>,
        )
        .mockReturnValueOnce(
          new Promise<any>((resolve) => {
            resolveFetches.push(resolve);
          }) as ReturnType<typeof fetchWithCache>,
        );

      const first = withCacheNamespace('scope-a', () =>
        provider.callApi('Concurrent sensitive prompt'),
      );
      const second = withCacheNamespace('scope-b', () =>
        provider.callApi('Concurrent sensitive prompt'),
      );

      await Promise.resolve();
      expect(fetchWithCache).toHaveBeenCalledTimes(2);

      resolveFetches[0]({
        data: {
          choices: [{ message: { content: 'Scoped output A' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });
      resolveFetches[1]({
        data: {
          choices: [{ message: { content: 'Scoped output B' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await expect(Promise.all([first, second])).resolves.toEqual([
        expect.objectContaining({ output: 'Scoped output A' }),
        expect.objectContaining({ output: 'Scoped output B' }),
      ]);
    });

    it('should avoid logging prompts and generated outputs in debug metadata', async () => {
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content: 'Generated secret response' } }],
          usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await provider.callApi('Sensitive prompt with sk-mistral-secret');

      const fetchCall = vi.mocked(fetchWithCache).mock.calls[0];
      expect(fetchCall[3]).toBe('json');
      expect(fetchCall[4]).toBe(true);
      expect(fetchCall[1]).toEqual(
        expect.objectContaining({
          headers: expect.objectContaining({
            'x-promptfoo-silent': 'true',
          }),
        }),
      );

      const debugLogs = JSON.stringify(vi.mocked(logger.debug).mock.calls);
      expect(debugLogs).not.toContain('Sensitive prompt');
      expect(debugLogs).not.toContain('sk-mistral-secret');
      expect(debugLogs).not.toContain('Generated secret response');
    });

    it('should not use cache when disabled', async () => {
      const mockResponse = {
        choices: [{ message: { content: 'Fresh output' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      vi.mocked(isCacheEnabled).mockImplementation(function () {
        return false;
      });
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result).toEqual({
        output: 'Fresh output',
        tokenUsage: {
          total: 10,
          prompt: 5,
          completion: 5,
          numRequests: 1,
        },
        cached: false,
        cost: expect.any(Number),
      });
      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        expect.stringContaining('/chat/completions'),
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });

    it('should handle API errors', async () => {
      vi.mocked(isCacheEnabled).mockImplementation(function () {
        return false;
      });
      const mockError = new Error('API Error');
      vi.mocked(fetchWithCache).mockRejectedValueOnce(mockError);

      const result = await provider.callApi('Test prompt');

      expect(result).toEqual({
        error: 'API call error: Error: API Error',
      });
    });

    it('should use custom API base URL if provided', async () => {
      const customProvider = new MistralChatCompletionProvider('mistral-tiny', {
        config: { apiBaseUrl: 'https://custom.mistral.ai/v1' },
      });
      vi.spyOn(customProvider, 'getApiKey').mockReturnValue('fake-api-key');
      const mockResponse = {
        choices: [{ message: { content: 'Custom API response' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await customProvider.callApi('Test prompt');

      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        'https://custom.mistral.ai/v1/chat/completions',
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });

    it('should use API host if provided', async () => {
      const customProvider = new MistralChatCompletionProvider('mistral-tiny', {
        config: { apiHost: 'custom.mistral.ai' },
      });
      vi.spyOn(customProvider, 'getApiKey').mockReturnValue('fake-api-key');
      const mockResponse = {
        choices: [{ message: { content: 'Custom API response' } }],
        usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await customProvider.callApi('Test prompt');

      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        'https://custom.mistral.ai/v1/chat/completions',
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });

    it('should return tool_calls as output when content is null (tool call response)', async () => {
      const mockToolCalls = [
        {
          id: 'call_123',
          type: 'function',
          function: { name: 'get_weather', arguments: '{"location":"Tokyo"}' },
        },
      ];
      const mockResponse = {
        choices: [{ message: { content: null, tool_calls: mockToolCalls } }],
        usage: { total_tokens: 20, prompt_tokens: 10, completion_tokens: 10 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toEqual(mockToolCalls);
      expect(result.error).toBeUndefined();
    });

    it('should return final text from chunked reasoning content', async () => {
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [
            {
              message: {
                content: [
                  {
                    type: 'thinking',
                    thinking: [{ type: 'text', text: 'Internal reasoning' }],
                  },
                  { type: 'text', text: 'Final answer' },
                ],
              },
            },
          ],
          usage: { total_tokens: 20, prompt_tokens: 10, completion_tokens: 10 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toBe('Final answer');
    });

    it('should preserve chunk arrays that contain non-reasoning metadata', async () => {
      const content = [
        { type: 'thinking', thinking: [{ type: 'text', text: 'Internal reasoning' }] },
        { type: 'text', text: 'Final answer' },
        { type: 'citation', url: 'https://example.com' },
      ];
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices: [{ message: { content } }],
          usage: { total_tokens: 20, prompt_tokens: 10, completion_tokens: 10 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toEqual(content);
    });

    it('should preserve all choices in metadata when n returns multiple completions', async () => {
      const choices = [
        { index: 0, message: { content: 'First response' } },
        { index: 1, message: { content: 'Second response' } },
      ];
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: {
          choices,
          usage: { total_tokens: 30, prompt_tokens: 10, completion_tokens: 20 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toBe('First response');
      expect(result.metadata?.choices).toEqual(choices);
    });

    it('should return full message when both content and tool_calls are present', async () => {
      const mockToolCalls = [
        {
          id: 'call_456',
          type: 'function',
          function: { name: 'search', arguments: '{"query":"test"}' },
        },
      ];
      const mockMessage = {
        content: 'I will search for that.',
        tool_calls: mockToolCalls,
      };
      const mockResponse = {
        choices: [{ message: mockMessage }],
        usage: { total_tokens: 20, prompt_tokens: 10, completion_tokens: 10 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toEqual(mockMessage);
      expect(result.error).toBeUndefined();
    });

    it('should return content when tool_calls is an empty array', async () => {
      const mockResponse = {
        choices: [{ message: { content: 'Test output', tool_calls: [] } }],
        usage: { total_tokens: 20, prompt_tokens: 10, completion_tokens: 10 },
      };
      vi.mocked(fetchWithCache).mockResolvedValueOnce({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callApi('Test prompt');

      expect(result.output).toBe('Test output');
      expect(result.error).toBeUndefined();
    });
  });

  describe('MistralEmbeddingProvider', () => {
    let provider: MistralEmbeddingProvider;

    beforeEach(() => {
      provider = new MistralEmbeddingProvider();
      vi.spyOn(provider, 'getApiKey').mockReturnValue('fake-api-key');
    });

    it('should create a provider with default options', () => {
      expect(provider.modelName).toBe('mistral-embed');
      expect(provider.config).toEqual({});
    });

    it('should support non-default embedding models such as codestral-embed', async () => {
      const codestralProvider = new MistralEmbeddingProvider({ modelName: 'codestral-embed' });
      vi.spyOn(codestralProvider, 'getApiKey').mockReturnValue('fake-api-key');
      expect(codestralProvider.modelName).toBe('codestral-embed');
      expect(codestralProvider.id()).toBe('mistral:embedding:codestral-embed');

      vi.mocked(fetchWithCache).mockResolvedValue({
        data: {
          model: 'codestral-embed',
          data: [{ embedding: [0.1, 0.2, 0.3] }],
          usage: { total_tokens: 1000, prompt_tokens: 1000 },
        },
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await codestralProvider.callEmbeddingApi('Test code');

      // Request sends the codestral-embed model
      const requestInit = vi.mocked(fetchWithCache).mock.calls[0]?.[1];
      const body = JSON.parse(requestInit?.body as string);
      expect(body.model).toBe('codestral-embed');
      // Codestral Embed input pricing is $0.15/1M: 1000 tokens => 0.00015
      expect(result.cost).toBeCloseTo(0.00015, 6);
    });

    it('should warn on an unknown embedding model', () => {
      const unknownProvider = new MistralEmbeddingProvider({ modelName: 'not-a-real-embed' });
      expect(unknownProvider.modelName).toBe('not-a-real-embed');
      expect(vi.mocked(logger.warn)).toHaveBeenCalledWith(
        expect.stringContaining('Using unknown Mistral embedding model'),
      );
    });

    it('should call Mistral Embedding API and return embedding with correct structure', async () => {
      const mockResponse = {
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValue({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callEmbeddingApi('Test text');

      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        expect.stringContaining('/embeddings'),
        expect.objectContaining({
          method: 'POST',
          headers: expect.objectContaining({
            'Content-Type': 'application/json',
            'x-promptfoo-silent': 'true',
            Authorization: expect.stringContaining('Bearer '),
          }),
          body: expect.stringContaining('"input":"Test text"'),
        }),
        expect.any(Number),
        'json',
        true,
      );

      expect(result).toEqual({
        embedding: [0.1, 0.2, 0.3],
        tokenUsage: {
          total: 5,
          prompt: 5,
          completion: 0,
          numRequests: 1,
        },
        cost: expect.closeTo(0.0000005, 0.0000001), // Approximately 5 tokens * 0.1 / 1000000
      });
    });

    it('should use cache for embedding when enabled', async () => {
      const mockResponse = {
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      };
      vi.mocked(isCacheEnabled).mockImplementation(function () {
        return true;
      });
      vi.mocked(fetchWithCache).mockResolvedValue({
        data: mockResponse,
        cached: true,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callEmbeddingApi('Test text');

      expect(result).toEqual({
        embedding: [0.1, 0.2, 0.3],
        tokenUsage: {
          total: 5,
          completion: 0,
          cached: 5,
          numRequests: 1,
        },
        cost: expect.closeTo(0.0000005, 0.0000001),
        cached: true,
      });
      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        expect.stringContaining('/embeddings'),
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });

    it('should not use cache for embedding when disabled', async () => {
      const mockResponse = {
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      };
      vi.mocked(isCacheEnabled).mockImplementation(function () {
        return false;
      });
      vi.mocked(fetchWithCache).mockResolvedValue({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      const result = await provider.callEmbeddingApi('Test text');

      expect(result).toEqual({
        embedding: [0.1, 0.2, 0.3],
        tokenUsage: {
          total: 5,
          prompt: 5,
          completion: 0,
          numRequests: 1,
        },
        cost: expect.any(Number),
      });
      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        expect.stringContaining('/embeddings'),
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });

    it('should hash embedding cache keys and avoid logging inputs or embeddings', async () => {
      const cacheGet = vi.fn().mockResolvedValue(null);
      const cacheSet = vi.fn();
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: cacheGet,
        set: cacheSet,
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);
      const mockResponse = {
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValue({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await provider.callEmbeddingApi('Sensitive embedding input sk-mistral-secret');

      const cacheKey = cacheGet.mock.calls[0]?.[0] as string;
      expect(cacheKey).toMatch(
        /^mistral:embedding:mistral-embed:[a-f0-9]{64}:[a-f0-9]{64}:[a-f0-9]{64}$/,
      );
      expect(cacheKey).not.toContain('Sensitive embedding input');
      expect(cacheKey).not.toContain('sk-mistral-secret');
      expect(cacheSet).toHaveBeenCalledWith(cacheKey, mockResponse);
      expect(vi.mocked(fetchWithCache).mock.calls[0][1]).toEqual(
        expect.objectContaining({
          headers: expect.objectContaining({
            'x-promptfoo-silent': 'true',
          }),
        }),
      );

      const debugLogs = JSON.stringify(vi.mocked(logger.debug).mock.calls);
      expect(debugLogs).not.toContain('Sensitive embedding input');
      expect(debugLogs).not.toContain('sk-mistral-secret');
      expect(debugLogs).not.toContain('[0.1,0.2,0.3]');
    });

    it('should return cached embeddings from hashed provider cache keys', async () => {
      const cacheGet = vi.fn().mockResolvedValue({
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      });
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: cacheGet,
        set: vi.fn(),
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);

      const result = await provider.callEmbeddingApi('Cached sensitive input');

      expect(fetchWithCache).not.toHaveBeenCalled();
      expect(result).toEqual({
        embedding: [0.1, 0.2, 0.3],
        tokenUsage: {
          total: 5,
          cached: 5,
          completion: 0,
          numRequests: 1,
        },
        cost: expect.closeTo(0.0000005, 0.0000001),
        cached: true,
      });
      const cacheKey = cacheGet.mock.calls[0]?.[0] as string;
      expect(cacheKey).not.toContain('Cached sensitive input');
    });

    it('should propagate cached embeddings through callApi', async () => {
      vi.mocked(isCacheEnabled).mockReturnValue(true);
      vi.mocked(getCache).mockReturnValue({
        get: vi.fn().mockResolvedValue({
          model: 'mistral-embed',
          data: [{ embedding: [0.1, 0.2, 0.3] }],
          usage: { total_tokens: 5, prompt_tokens: 5 },
        }),
        set: vi.fn(),
        wrap: vi.fn(),
        del: vi.fn(),
        clear: vi.fn(),
        stores: [
          {
            get: vi.fn(),
            set: vi.fn(),
          },
        ] as any,
        mget: vi.fn(),
        mset: vi.fn(),
        mdel: vi.fn(),
        reset: vi.fn(),
        ttl: vi.fn(),
        on: vi.fn(),
        removeAllListeners: vi.fn(),
      } as any);

      const result = await provider.callApi('Cached sensitive input');

      expect(fetchWithCache).not.toHaveBeenCalled();
      expect(result).toEqual({
        output: JSON.stringify([0.1, 0.2, 0.3]),
        tokenUsage: {
          total: 5,
          cached: 5,
          completion: 0,
          numRequests: 1,
        },
        cost: expect.closeTo(0.0000005, 0.0000001),
        cached: true,
      });
    });

    it('should handle API errors', async () => {
      const mockError = new Error('API Error');
      vi.mocked(fetchWithCache).mockRejectedValue(mockError);

      await expect(provider.callEmbeddingApi('Test text')).rejects.toThrow('API Error');
    });

    it('should return provider id and string representation', () => {
      expect(provider.id()).toBe('mistral:embedding:mistral-embed');
      expect(provider.toString()).toBe('[Mistral Embedding Provider mistral-embed]');
    });

    it('should use custom API base URL if provided', async () => {
      const customProvider = new MistralEmbeddingProvider({
        config: { apiBaseUrl: 'https://custom.mistral.ai/v1' },
      });
      vi.spyOn(customProvider, 'getApiKey').mockReturnValue('fake-api-key');
      const mockResponse = {
        model: 'mistral-embed',
        data: [{ embedding: [0.1, 0.2, 0.3] }],
        usage: { total_tokens: 5, prompt_tokens: 5 },
      };
      vi.mocked(fetchWithCache).mockResolvedValue({
        data: mockResponse,
        cached: false,
        status: 200,
        statusText: 'OK',
      });

      await customProvider.callEmbeddingApi('Test text');

      expect(vi.mocked(fetchWithCache)).toHaveBeenCalledWith(
        'https://custom.mistral.ai/v1/embeddings',
        expect.any(Object),
        expect.any(Number),
        'json',
        true,
      );
    });
  });
});