promptfoo--promptfoo/test/prompts/index.test.ts

import * as fs from 'fs';

import dedent from 'dedent';
import { globSync } from 'glob';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { importModule } from '../../src/esm';
import { processPrompts, readPrompts, readProviderPromptMap } from '../../src/prompts/index';
import { maybeFilePath } from '../../src/prompts/utils';
import { mockProcessEnv } from '../util/utils';

import type {
  ApiProvider,
  EvaluateTestSuite,
  Prompt,
  ProviderResponse,
} from '../../src/types/index';

vi.mock('proxy-agent', () => ({
  ProxyAgent: vi.fn().mockImplementation(() => ({})),
}));

vi.mock('glob', async () => {
  const actual = await vi.importActual<typeof import('glob')>('glob');
  return {
    ...actual,
    globSync: vi.fn().mockImplementation(actual.globSync),
  };
});

vi.mock('fs');

vi.mock('python-shell');
vi.mock('../../src/esm', async () => {
  const actual = await vi.importActual<typeof import('../../src/esm')>('../../src/esm');
  return {
    ...actual,
    importModule: vi.fn().mockImplementation(actual.importModule),
  };
});
vi.mock('../../src/python/wrapper');
vi.mock('../../src/prompts/utils', async () => {
  const actual =
    await vi.importActual<typeof import('../../src/prompts/utils')>('../../src/prompts/utils');
  return {
    ...actual,
    maybeFilePath: vi.fn().mockImplementation(actual.maybeFilePath),
  };
});

describe('readPrompts', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });
  afterEach(() => {
    mockProcessEnv({ PROMPTFOO_STRICT_FILES: undefined });
    vi.mocked(fs.readFileSync).mockReset();
    vi.mocked(fs.statSync).mockReset();
    vi.mocked(globSync).mockReset();
    vi.mocked(maybeFilePath).mockClear();
  });

  it('should throw an error for invalid inputs', async () => {
    await expect(readPrompts(null as any)).rejects.toThrow('Invalid input prompt: null');
    await expect(readPrompts(undefined as any)).rejects.toThrow('Invalid input prompt: undefined');
    await expect(readPrompts(1 as any)).rejects.toThrow('Invalid input prompt: 1');
    await expect(readPrompts(true as any)).rejects.toThrow('Invalid input prompt: true');
    await expect(readPrompts(false as any)).rejects.toThrow('Invalid input prompt: false');
  });

  it('should throw an error for empty inputs', async () => {
    await expect(readPrompts([])).rejects.toThrow('Invalid input prompt: []');
    await expect(readPrompts({} as any)).rejects.toThrow('Invalid input prompt: {}');
    await expect(readPrompts('')).rejects.toThrow('Invalid input prompt: ""');
  });

  it('should throw an error when PROMPTFOO_STRICT_FILES is true and the file does not exist', async () => {
    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(fs.readFileSync).mockImplementationOnce(() => {
      throw new Error("ENOENT: no such file or directory, stat 'non-existent-file.txt'");
    });
    await expect(readPrompts('non-existent-file.txt')).rejects.toThrow(
      expect.objectContaining({
        message: expect.stringMatching(
          /ENOENT: no such file or directory, stat '.*non-existent-file.txt'/,
        ),
      }),
    );
  });

  it('should throw an error for a .txt file with no prompts', async () => {
    vi.mocked(fs.readFileSync).mockReturnValueOnce('');
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
    await expect(readPrompts(['prompts.txt'])).rejects.toThrow(
      'There are no prompts in "prompts.txt"',
    );
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
  });

  it('should throw an error for an unsupported file format', async () => {
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
    await expect(readPrompts(['unsupported.for.mat'])).rejects.toThrow(
      'There are no prompts in "unsupported.for.mat"',
    );
    expect(fs.readFileSync).toHaveBeenCalledTimes(0);
  });

  it('should read a single prompt', async () => {
    const prompt = 'This is a test prompt';
    await expect(readPrompts(prompt)).resolves.toEqual([
      {
        raw: prompt,
        label: prompt,
      },
    ]);
  });

  it('should read a list of prompts', async () => {
    const prompts = ['Sample prompt A', 'Sample prompt B'];
    await expect(readPrompts(prompts)).resolves.toEqual([
      {
        raw: 'Sample prompt A',
        label: 'Sample prompt A',
      },
      {
        raw: 'Sample prompt B',
        label: 'Sample prompt B',
      },
    ]);
  });

  it('should read a .txt file with a single prompt', async () => {
    vi.mocked(fs.readFileSync).mockReturnValueOnce('Sample Prompt');
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    await expect(readPrompts('prompts.txt')).resolves.toEqual([
      {
        label: 'prompts.txt: Sample Prompt',
        raw: 'Sample Prompt',
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
  });

  it.each([
    ['prompts.txt'],
    'prompts.txt',
  ])(`should read a single prompt file with input:%p`, async (promptPath) => {
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(fs.readFileSync).mockReturnValue('Test prompt 1\n---\nTest prompt 2');
    vi.mocked(globSync).mockImplementation((pathOrGlob) => [pathOrGlob.toString()]);
    await expect(readPrompts(promptPath)).resolves.toEqual([
      {
        label: 'prompts.txt: Test prompt 1',
        raw: 'Test prompt 1',
      },
      {
        label: 'prompts.txt: Test prompt 2',
        raw: 'Test prompt 2',
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
  });

  it('should read multiple prompt files', async () => {
    vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
      if (filePath.toString().endsWith('prompt1.txt')) {
        return 'Test prompt 1\n---\nTest prompt 2';
      } else if (filePath.toString().endsWith('prompt2.txt')) {
        return 'Test prompt 3\n---\nTest prompt 4\n---\nTest prompt 5';
      }
      throw new Error(`Unexpected file path in test: ${filePath}`);
    });
    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
    vi.mocked(globSync).mockImplementation((pathOrGlob) => [pathOrGlob.toString()]);
    await expect(readPrompts(['prompt1.txt', 'prompt2.txt'])).resolves.toEqual([
      {
        label: 'prompt1.txt: Test prompt 1',
        raw: 'Test prompt 1',
      },
      {
        label: 'prompt1.txt: Test prompt 2',
        raw: 'Test prompt 2',
      },
      {
        label: 'prompt2.txt: Test prompt 3',
        raw: 'Test prompt 3',
      },
      {
        label: 'prompt2.txt: Test prompt 4',
        raw: 'Test prompt 4',
      },
      {
        label: 'prompt2.txt: Test prompt 5',
        raw: 'Test prompt 5',
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
  });

  it('should read with map input', async () => {
    vi.mocked(fs.readFileSync).mockReturnValue('some raw text');
    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
    await expect(
      readPrompts({
        'prompts.txt': 'foo1',
        'prompts2.txt': 'foo2',
      }),
    ).resolves.toEqual([
      { raw: 'some raw text', label: 'foo1: prompts.txt: some raw text' },
      { raw: 'some raw text', label: 'foo2: prompts2.txt: some raw text' },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
  });

  it('should read a .json file', async () => {
    const mockJsonContent = JSON.stringify([
      { name: 'You are a helpful assistant', role: 'system' },
      { name: 'How do I get to the moon?', role: 'user' },
    ]);

    vi.mocked(fs.readFileSync).mockReturnValueOnce(mockJsonContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const filePath = 'file://path/to/mock.json';
    await expect(readPrompts([filePath])).resolves.toEqual([
      {
        raw: mockJsonContent,
        label: expect.stringContaining(`mock.json: ${mockJsonContent}`),
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('mock.json'), 'utf8');
    expect(fs.statSync).toHaveBeenCalledTimes(1);
  });

  it('should read a .jsonl file', async () => {
    const data = [
      [
        { role: 'system', content: 'You are a helpful assistant.' },
        { role: 'user', content: 'Who won the world series in {{ year }}?' },
      ],
      [
        { role: 'system', content: 'You are a helpful assistant.' },
        { role: 'user', content: 'Who won the superbowl in {{ year }}?' },
      ],
    ];

    vi.mocked(fs.readFileSync).mockReturnValueOnce(data.map((o) => JSON.stringify(o)).join('\n'));
    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
    await expect(readPrompts(['prompts.jsonl'])).resolves.toEqual([
      {
        raw: JSON.stringify(data[0]),
        label: `prompts.jsonl: ${JSON.stringify(data[0])}`,
      },
      {
        raw: JSON.stringify(data[1]),
        label: `prompts.jsonl: ${JSON.stringify(data[1])}`,
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
  });

  const yamlContent = dedent`
    - role: user
      content:
        - type: text
          text: "What's in this image?"
        - type: image_url
          image_url:
            url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"`;

  it('should read a .yaml file', async () => {
    const expectedJson = JSON.stringify([
      {
        role: 'user',
        content: [
          { type: 'text', text: "What's in this image?" },
          {
            type: 'image_url',
            image_url: {
              url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
            },
          },
        ],
      },
    ]);

    vi.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    await expect(readPrompts('prompts.yaml')).resolves.toEqual([
      {
        raw: expectedJson,
        label: expect.stringContaining(
          'prompts.yaml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}',
        ),
        config: undefined,
      },
    ]);
  });

  it('should read a .yml file', async () => {
    const expectedJson = JSON.stringify([
      {
        role: 'user',
        content: [
          { type: 'text', text: "What's in this image?" },
          {
            type: 'image_url',
            image_url: {
              url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
            },
          },
        ],
      },
    ]);

    vi.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    await expect(readPrompts('image-summary.yml')).resolves.toEqual([
      {
        raw: expectedJson,
        label: expect.stringContaining(
          'image-summary.yml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}',
        ),
        config: undefined,
      },
    ]);
  });

  it('should read a markdown file', async () => {
    const mdContent = '# Test Heading\n\nThis is a test markdown file.';
    vi.mocked(fs.readFileSync).mockReturnValueOnce(mdContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);

    await expect(readPrompts('test.md')).resolves.toEqual([
      {
        raw: mdContent,
        label: 'test.md: # Test Heading\n\nThis is a test markdown file....',
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
    expect(fs.readFileSync).toHaveBeenCalledWith('test.md', 'utf8');
  });

  it('should read a Jinja2 file', async () => {
    const jinjaContent =
      'You are a helpful assistant.\nPlease answer the following question about {{ topic }}: {{ question }}';
    vi.mocked(fs.readFileSync).mockReturnValueOnce(jinjaContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);

    const result = await readPrompts('template.j2');

    // Check that we get a result with the right content
    expect(result).toHaveLength(1);
    expect(result[0].raw).toEqual(jinjaContent);
    expect(result[0].config).toBeUndefined();

    // Check that the label contains the expected text but don't test exact truncation
    expect(result[0].label).toContain('template.j2: You are a helpful assistant.');

    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
    expect(fs.readFileSync).toHaveBeenCalledWith('template.j2', 'utf8');
  });

  it('should read a .py prompt object array', async () => {
    const prompts = [
      { id: 'prompts.py:prompt1', label: 'First prompt' },
      { id: 'prompts.py:prompt2', label: 'Second prompt' },
    ];

    const code = dedent`
      def prompt1:
        return 'First prompt'
      def prompt2:
        return 'Second prompt'
      `;
    vi.mocked(fs.readFileSync).mockReturnValue(code);
    await expect(readPrompts(prompts)).resolves.toEqual([
      {
        raw: code,
        label: 'First prompt',
        function: expect.any(Function),
      },
      {
        raw: code,
        label: 'Second prompt',
        function: expect.any(Function),
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
  });

  it('should read a .py file', async () => {
    const code = `print('dummy prompt')`;
    vi.mocked(fs.readFileSync).mockReturnValue(code);
    await expect(readPrompts('prompt.py')).resolves.toEqual([
      {
        function: expect.any(Function),
        label: `prompt.py: ${code}`,
        raw: code,
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
  });

  it('should read a .js file without named function', async () => {
    const promptPath = 'prompt.js';
    const mockFunction = () => console.log('dummy prompt');

    vi.mocked(importModule).mockResolvedValueOnce(mockFunction);
    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);

    await expect(readPrompts(promptPath)).resolves.toEqual([
      {
        raw: expect.stringMatching(/\(\)\s*=>\s*console\.log\(['"]dummy prompt['"]\)/),
        label: 'prompt.js',
        function: expect.any(Function),
        config: {},
      },
    ]);
    expect(importModule).toHaveBeenCalledWith(promptPath, undefined);
    expect(fs.statSync).toHaveBeenCalledTimes(1);
  });

  it('should read a .js file with named function', async () => {
    const promptPath = 'prompt.js:functionName';
    const mockFunction = (_context: {
      vars: Record<string, string | object>;
      provider?: ApiProvider;
    }) => 'dummy prompt result';

    vi.mocked(importModule).mockResolvedValueOnce(mockFunction);
    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);

    const result = await readPrompts(promptPath);
    expect(result).toEqual([
      {
        raw: String(mockFunction),
        label: 'prompt.js:functionName',
        function: expect.any(Function),
        config: {},
      },
    ]);

    const promptFunction = result[0].function as unknown as (context: {
      vars: Record<string, string | object>;
      provider?: ApiProvider;
    }) => string;
    expect(promptFunction({ vars: {}, provider: { id: () => 'foo' } as ApiProvider })).toBe(
      'dummy prompt result',
    );

    expect(importModule).toHaveBeenCalledWith('prompt.js', 'functionName');
    expect(fs.statSync).toHaveBeenCalledTimes(1);
  });

  it('should read a directory', async () => {
    vi.mocked(fs.statSync).mockImplementation((filePath) => {
      if (filePath.toString().endsWith('prompt1.txt')) {
        return { isDirectory: () => false } as fs.Stats;
      } else if (filePath.toString().endsWith('prompts')) {
        return { isDirectory: () => true } as fs.Stats;
      }
      throw new Error(`Unexpected file path in test: ${filePath}`);
    });
    vi.mocked(globSync).mockImplementation(() => ['prompt1.txt', 'prompt2.txt']);
    // The mocked paths here are an artifact of our globSync mock. In a real
    // world setting we would get back `prompts/prompt1.txt` instead of `prompts/*/prompt1.txt`
    // but for the sake of this test we are just going to pretend that the globSync
    // mock is doing the right thing and giving us back the right paths.
    vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
      if (
        filePath.toString().endsWith('prompt1.txt') ||
        filePath.toString().endsWith('*/prompt1.txt')
      ) {
        return 'Test prompt 1\n---\nTest prompt 2';
      } else if (
        filePath.toString().endsWith('prompt2.txt') ||
        filePath.toString().endsWith('*/prompt2.txt')
      ) {
        return 'Test prompt 3\n---\nTest prompt 4\n---\nTest prompt 5';
      }
      throw new Error(`Unexpected file path in test: ${filePath}`);
    });
    await expect(readPrompts(['prompts/*'])).resolves.toEqual([
      {
        label: expect.stringMatching('prompt1.txt: Test prompt 1'),
        raw: 'Test prompt 1',
      },
      {
        label: expect.stringMatching('prompt1.txt: Test prompt 2'),
        raw: 'Test prompt 2',
      },
      {
        label: expect.stringMatching('prompt2.txt: Test prompt 3'),
        raw: 'Test prompt 3',
      },
      {
        label: expect.stringMatching('prompt2.txt: Test prompt 4'),
        raw: 'Test prompt 4',
      },
      {
        label: expect.stringMatching('prompt2.txt: Test prompt 5'),
        raw: 'Test prompt 5',
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
    expect(fs.statSync).toHaveBeenCalledTimes(3);
  });

  it('should fall back to a string if maybeFilePath is true but a file does not exist', async () => {
    vi.mocked(globSync).mockReturnValueOnce([]);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
    await expect(readPrompts('non-existent-file.txt*')).resolves.toEqual([
      { raw: 'non-existent-file.txt*', label: 'non-existent-file.txt*' },
    ]);
  });

  it('should handle a prompt with a function', async () => {
    const promptWithFunction: Partial<Prompt> = {
      raw: 'dummy raw text',
      label: 'Function Prompt',
      function: vi.fn().mockResolvedValue('Hello, world!'),
    };

    await expect(readPrompts([promptWithFunction])).resolves.toEqual([
      {
        raw: 'dummy raw text',
        label: 'Function Prompt',
        function: expect.any(Function),
      },
    ]);
    expect(promptWithFunction.function).not.toHaveBeenCalled();
  });
  it('should read a single-column CSV file with header', async () => {
    const csvContent = dedent`
      prompt
      Tell me about {{topic}}
      Explain {{topic}} in simple terms
      Write a poem about {{topic}}
    `;
    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await readPrompts(['test.csv']);

    expect(result).toHaveLength(3);
    expect(result[0]).toEqual({
      raw: 'Tell me about {{topic}}',
      label: 'Prompt 1 - Tell me about {{topic}}',
    });
    expect(result[1]).toEqual({
      raw: 'Explain {{topic}} in simple terms',
      label: 'Prompt 2 - Explain {{topic}} in simple terms',
    });
    expect(result[2]).toEqual({
      raw: 'Write a poem about {{topic}}',
      label: 'Prompt 3 - Write a poem about {{topic}}',
    });
  });

  it('should read a single-column CSV file without header', async () => {
    const csvContent = dedent`
      Tell me about {{topic}}
      Explain {{topic}} in simple terms
      Write a poem about {{topic}}
    `;
    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await readPrompts(['test.csv']);

    expect(result).toHaveLength(3);
    expect(result[0]).toEqual({
      raw: 'Tell me about {{topic}}',
      label: 'Prompt 1 - Tell me about {{topic}}',
    });
    expect(result[1]).toEqual({
      raw: 'Explain {{topic}} in simple terms',
      label: 'Prompt 2 - Explain {{topic}} in simple terms',
    });
    expect(result[2]).toEqual({
      raw: 'Write a poem about {{topic}}',
      label: 'Prompt 3 - Write a poem about {{topic}}',
    });
  });

  it('should read a two-column CSV file with prompt and label', async () => {
    const csvContent = dedent`
      prompt,label
      Tell me about {{topic}},Basic Query
      Explain {{topic}} in simple terms,Simple Explanation
      Write a poem about {{topic}},Poetry Generator
    `;
    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await readPrompts(['test.csv']);

    expect(result).toHaveLength(3);
    expect(result[0]).toEqual({
      raw: 'Tell me about {{topic}}',
      label: 'Basic Query',
    });
    expect(result[1]).toEqual({
      raw: 'Explain {{topic}} in simple terms',
      label: 'Simple Explanation',
    });
    expect(result[2]).toEqual({
      raw: 'Write a poem about {{topic}}',
      label: 'Poetry Generator',
    });
  });
});

describe('readProviderPromptMap', () => {
  let config: Pick<Partial<EvaluateTestSuite>, 'providers'>;
  let parsedPrompts: Prompt[];

  beforeEach(() => {
    vi.clearAllMocks();
    parsedPrompts = [
      { label: 'prompt1', raw: 'prompt1' },
      { label: 'prompt2', raw: 'prompt2' },
    ];
  });

  it('should return an empty object if config.providers is undefined', () => {
    config = {};
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({});
  });

  it('should return a map with all prompts if config.providers is a string', () => {
    config = { providers: 'provider1' };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['prompt1', 'prompt2'],
    });
  });

  it('should return a map with all prompts if config.providers is a function', () => {
    config = {
      providers: () =>
        Promise.resolve({
          providerName: 'Custom function',
          prompts: ['prompt1', 'prompt2'],
        }) as Promise<ProviderResponse>,
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      'Custom function': ['prompt1', 'prompt2'],
    });
  });

  it('should handle provider objects with id and prompts', () => {
    config = {
      providers: [{ id: 'provider1', prompts: ['customPrompt1'] }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
  });

  it('should handle a top-level ApiProvider object', () => {
    const provider: ApiProvider = {
      id: () => 'provider1',
      callApi: vi.fn<ApiProvider['callApi']>(),
      label: 'providerLabel',
    };
    config = { providers: provider };

    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['prompt1', 'prompt2'],
    });
  });

  it('should handle ApiProvider objects in provider arrays', () => {
    const provider: ApiProvider = {
      id: () => 'provider1',
      callApi: vi.fn<ApiProvider['callApi']>(),
      label: 'providerLabel',
    };
    config = { providers: [provider] };

    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['prompt1', 'prompt2'],
      providerLabel: ['prompt1', 'prompt2'],
    });
  });

  it('should handle provider objects with id, label, and prompts', () => {
    config = {
      providers: [{ id: 'provider1', label: 'providerLabel', prompts: ['customPrompt1'] }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['customPrompt1'],
      providerLabel: ['customPrompt1'],
    });
  });

  it('should handle provider options map with id and prompts', () => {
    config = {
      providers: [
        {
          originalProvider: {
            id: 'provider1',
            prompts: ['customPrompt1'],
          },
        },
      ],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
  });

  it('should handle provider options map without id and use original id', () => {
    config = {
      providers: [
        {
          originalProvider: {
            prompts: ['customPrompt1'],
          },
        },
      ],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      originalProvider: ['customPrompt1'],
    });
  });

  it('should use rawProvider.prompts if provided for provider objects with id', () => {
    config = {
      providers: [{ id: 'provider1', prompts: ['customPrompt1'] }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
  });

  it('should fall back to allPrompts if no prompts provided for provider objects with id', () => {
    config = {
      providers: [{ id: 'provider1' }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['prompt1', 'prompt2'],
    });
  });

  it('should use rawProvider.prompts for both id and label if provided', () => {
    config = {
      providers: [{ id: 'provider1', label: 'providerLabel', prompts: ['customPrompt1'] }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['customPrompt1'],
      providerLabel: ['customPrompt1'],
    });
  });

  it('should fall back to allPrompts for both id and label if no prompts provided', () => {
    config = {
      providers: [{ id: 'provider1', label: 'providerLabel' }],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      provider1: ['prompt1', 'prompt2'],
      providerLabel: ['prompt1', 'prompt2'],
    });
  });

  it('should use providerObject.id from ProviderOptionsMap when provided', () => {
    config = {
      providers: [
        {
          originalProvider: {
            id: 'explicitId',
            prompts: ['customPrompt1'],
          },
        },
      ],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ explicitId: ['customPrompt1'] });
  });

  it('should fallback to originalId when providerObject.id is not specified in ProviderOptionsMap', () => {
    config = {
      providers: [
        {
          originalProvider: {
            // 'originalProvider' is treated as originalId
            prompts: ['customPrompt1'],
          },
        },
      ],
    };
    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
      originalProvider: ['customPrompt1'],
    });
  });
});

describe('processPrompts', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('should process function prompts', async () => {
    function testFunction(vars: any) {
      return `Hello ${vars.name}`;
    }

    const result = await processPrompts([testFunction]);

    expect(result).toEqual([
      {
        raw: testFunction.toString(),
        label: 'testFunction',
        function: testFunction,
      },
    ]);
  });

  it('should process function prompts with no name', async () => {
    function promptFn(vars: any) {
      return `Hello ${vars.name}`;
    }
    Object.defineProperty(promptFn, 'name', { value: '' });

    const result = await processPrompts([promptFn]);

    expect(result).toEqual([
      {
        raw: promptFn.toString(),
        label: '',
        function: expect.any(Function),
      },
    ]);
    expect(fs.readFileSync).toHaveBeenCalledTimes(0);
  });

  it('should process string prompts by calling readPrompts', async () => {
    vi.mocked(fs.readFileSync).mockReturnValueOnce('test prompt');
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await processPrompts(['test.txt']);

    expect(result).toEqual([
      {
        raw: 'test prompt',
        label: 'test.txt: test prompt',
      },
    ]);
  });

  it('should process Jinja2 files', async () => {
    const jinjaContent =
      'You are a helpful assistant for {{ user }}.\nPlease provide information about {{ topic }}.';
    vi.mocked(fs.readFileSync).mockReturnValueOnce(jinjaContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
    vi.mocked(maybeFilePath).mockReturnValueOnce(true);

    const result = await processPrompts(['template.j2']);

    // Check that we get a result with the right content
    expect(result).toHaveLength(1);
    expect(result[0].raw).toEqual(jinjaContent);
    expect(result[0].config).toBeUndefined();

    // Check that the label contains the expected text but don't test exact truncation
    expect(result[0].label).toContain('template.j2: You are a helpful assistant for {{ user }}.');

    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
    expect(fs.readFileSync).toHaveBeenCalledWith('template.j2', 'utf8');
  });

  it('should process valid prompt schema objects', async () => {
    const validPrompt = {
      raw: 'test prompt',
      label: 'test label',
    };

    const result = await processPrompts([validPrompt]);

    expect(result).toEqual([validPrompt]);
  });

  it('should fall back to JSON serialization for invalid prompt schema objects', async () => {
    const invalidPrompt = {
      invalidField: 'some value',
      anotherField: 123,
    };

    const result = await processPrompts([invalidPrompt]);

    expect(result).toEqual([
      {
        raw: JSON.stringify(invalidPrompt),
        label: JSON.stringify(invalidPrompt),
      },
    ]);
  });

  it('should process multiple prompts of different types', async () => {
    function testFunction(vars: any) {
      return `Hello ${vars.name}`;
    }

    const validPrompt = {
      raw: 'test prompt',
      label: 'test label',
    };

    vi.mocked(fs.readFileSync).mockReturnValueOnce('file prompt');
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await processPrompts([testFunction, 'test.txt', validPrompt]);

    expect(result).toEqual([
      {
        raw: testFunction.toString(),
        label: 'testFunction',
        function: testFunction,
      },
      {
        raw: 'file prompt',
        label: 'test.txt: file prompt',
      },
      validPrompt,
    ]);
  });

  it('should process CSV files', async () => {
    const csvContent = `prompt,label
Tell me about {{topic}},Basic Query
Explain {{topic}} in simple terms,Simple Explanation`;
    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await processPrompts(['test.csv']);

    expect(result).toEqual([
      {
        raw: 'Tell me about {{topic}}',
        label: 'Basic Query',
      },
      {
        raw: 'Explain {{topic}} in simple terms',
        label: 'Simple Explanation',
      },
    ]);
  });

  it('should flatten array results from readPrompts', async () => {
    vi.mocked(fs.readFileSync).mockReturnValueOnce('prompt1\n---\nprompt2');
    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);

    const result = await processPrompts(['test.txt']);

    expect(result).toEqual([
      { raw: 'prompt1', label: 'test.txt: prompt1' },
      { raw: 'prompt2', label: 'test.txt: prompt2' },
    ]);
    expect(Array.isArray(result)).toBe(true);
    expect(result).toHaveLength(2);
  });
});