chore: import upstream snapshot with attribution

test/prompts/constants.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+import { VALID_FILE_EXTENSIONS } from '../../src/prompts/constants';
+
+describe('VALID_FILE_EXTENSIONS', () => {
+  it('should include .j2 extension', () => {
+    expect(VALID_FILE_EXTENSIONS).toContain('.j2');
+  });
+
+  it('should include common file extensions', () => {
+    expect(VALID_FILE_EXTENSIONS).toContain('.js');
+    expect(VALID_FILE_EXTENSIONS).toContain('.json');
+    expect(VALID_FILE_EXTENSIONS).toContain('.md');
+    expect(VALID_FILE_EXTENSIONS).toContain('.txt');
+    expect(VALID_FILE_EXTENSIONS).toContain('.yaml');
+    expect(VALID_FILE_EXTENSIONS).toContain('.yml');
+  });
+});

test/prompts/index.test.ts

@@ -0,0 +1,964 @@
+import * as fs from 'fs';
+
+import dedent from 'dedent';
+import { globSync } from 'glob';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { importModule } from '../../src/esm';
+import { processPrompts, readPrompts, readProviderPromptMap } from '../../src/prompts/index';
+import { maybeFilePath } from '../../src/prompts/utils';
+import { mockProcessEnv } from '../util/utils';
+
+import type {
+  ApiProvider,
+  EvaluateTestSuite,
+  Prompt,
+  ProviderResponse,
+} from '../../src/types/index';
+
+vi.mock('proxy-agent', () => ({
+  ProxyAgent: vi.fn().mockImplementation(() => ({})),
+}));
+
+vi.mock('glob', async () => {
+  const actual = await vi.importActual<typeof import('glob')>('glob');
+  return {
+    ...actual,
+    globSync: vi.fn().mockImplementation(actual.globSync),
+  };
+});
+
+vi.mock('fs');
+
+vi.mock('python-shell');
+vi.mock('../../src/esm', async () => {
+  const actual = await vi.importActual<typeof import('../../src/esm')>('../../src/esm');
+  return {
+    ...actual,
+    importModule: vi.fn().mockImplementation(actual.importModule),
+  };
+});
+vi.mock('../../src/python/wrapper');
+vi.mock('../../src/prompts/utils', async () => {
+  const actual =
+    await vi.importActual<typeof import('../../src/prompts/utils')>('../../src/prompts/utils');
+  return {
+    ...actual,
+    maybeFilePath: vi.fn().mockImplementation(actual.maybeFilePath),
+  };
+});
+
+describe('readPrompts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+  afterEach(() => {
+    mockProcessEnv({ PROMPTFOO_STRICT_FILES: undefined });
+    vi.mocked(fs.readFileSync).mockReset();
+    vi.mocked(fs.statSync).mockReset();
+    vi.mocked(globSync).mockReset();
+    vi.mocked(maybeFilePath).mockClear();
+  });
+
+  it('should throw an error for invalid inputs', async () => {
+    await expect(readPrompts(null as any)).rejects.toThrow('Invalid input prompt: null');
+    await expect(readPrompts(undefined as any)).rejects.toThrow('Invalid input prompt: undefined');
+    await expect(readPrompts(1 as any)).rejects.toThrow('Invalid input prompt: 1');
+    await expect(readPrompts(true as any)).rejects.toThrow('Invalid input prompt: true');
+    await expect(readPrompts(false as any)).rejects.toThrow('Invalid input prompt: false');
+  });
+
+  it('should throw an error for empty inputs', async () => {
+    await expect(readPrompts([])).rejects.toThrow('Invalid input prompt: []');
+    await expect(readPrompts({} as any)).rejects.toThrow('Invalid input prompt: {}');
+    await expect(readPrompts('')).rejects.toThrow('Invalid input prompt: ""');
+  });
+
+  it('should throw an error when PROMPTFOO_STRICT_FILES is true and the file does not exist', async () => {
+    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(fs.readFileSync).mockImplementationOnce(() => {
+      throw new Error("ENOENT: no such file or directory, stat 'non-existent-file.txt'");
+    });
+    await expect(readPrompts('non-existent-file.txt')).rejects.toThrow(
+      expect.objectContaining({
+        message: expect.stringMatching(
+          /ENOENT: no such file or directory, stat '.*non-existent-file.txt'/,
+        ),
+      }),
+    );
+  });
+
+  it('should throw an error for a .txt file with no prompts', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValueOnce('');
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
+    await expect(readPrompts(['prompts.txt'])).rejects.toThrow(
+      'There are no prompts in "prompts.txt"',
+    );
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should throw an error for an unsupported file format', async () => {
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+    mockProcessEnv({ PROMPTFOO_STRICT_FILES: 'true' });
+    await expect(readPrompts(['unsupported.for.mat'])).rejects.toThrow(
+      'There are no prompts in "unsupported.for.mat"',
+    );
+    expect(fs.readFileSync).toHaveBeenCalledTimes(0);
+  });
+
+  it('should read a single prompt', async () => {
+    const prompt = 'This is a test prompt';
+    await expect(readPrompts(prompt)).resolves.toEqual([
+      {
+        raw: prompt,
+        label: prompt,
+      },
+    ]);
+  });
+
+  it('should read a list of prompts', async () => {
+    const prompts = ['Sample prompt A', 'Sample prompt B'];
+    await expect(readPrompts(prompts)).resolves.toEqual([
+      {
+        raw: 'Sample prompt A',
+        label: 'Sample prompt A',
+      },
+      {
+        raw: 'Sample prompt B',
+        label: 'Sample prompt B',
+      },
+    ]);
+  });
+
+  it('should read a .txt file with a single prompt', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValueOnce('Sample Prompt');
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    await expect(readPrompts('prompts.txt')).resolves.toEqual([
+      {
+        label: 'prompts.txt: Sample Prompt',
+        raw: 'Sample Prompt',
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+  });
+
+  it.each([
+    ['prompts.txt'],
+    'prompts.txt',
+  ])(`should read a single prompt file with input:%p`, async (promptPath) => {
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(fs.readFileSync).mockReturnValue('Test prompt 1\n---\nTest prompt 2');
+    vi.mocked(globSync).mockImplementation((pathOrGlob) => [pathOrGlob.toString()]);
+    await expect(readPrompts(promptPath)).resolves.toEqual([
+      {
+        label: 'prompts.txt: Test prompt 1',
+        raw: 'Test prompt 1',
+      },
+      {
+        label: 'prompts.txt: Test prompt 2',
+        raw: 'Test prompt 2',
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should read multiple prompt files', async () => {
+    vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
+      if (filePath.toString().endsWith('prompt1.txt')) {
+        return 'Test prompt 1\n---\nTest prompt 2';
+      } else if (filePath.toString().endsWith('prompt2.txt')) {
+        return 'Test prompt 3\n---\nTest prompt 4\n---\nTest prompt 5';
+      }
+      throw new Error(`Unexpected file path in test: ${filePath}`);
+    });
+    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(globSync).mockImplementation((pathOrGlob) => [pathOrGlob.toString()]);
+    await expect(readPrompts(['prompt1.txt', 'prompt2.txt'])).resolves.toEqual([
+      {
+        label: 'prompt1.txt: Test prompt 1',
+        raw: 'Test prompt 1',
+      },
+      {
+        label: 'prompt1.txt: Test prompt 2',
+        raw: 'Test prompt 2',
+      },
+      {
+        label: 'prompt2.txt: Test prompt 3',
+        raw: 'Test prompt 3',
+      },
+      {
+        label: 'prompt2.txt: Test prompt 4',
+        raw: 'Test prompt 4',
+      },
+      {
+        label: 'prompt2.txt: Test prompt 5',
+        raw: 'Test prompt 5',
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+  });
+
+  it('should read with map input', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValue('some raw text');
+    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
+    await expect(
+      readPrompts({
+        'prompts.txt': 'foo1',
+        'prompts2.txt': 'foo2',
+      }),
+    ).resolves.toEqual([
+      { raw: 'some raw text', label: 'foo1: prompts.txt: some raw text' },
+      { raw: 'some raw text', label: 'foo2: prompts2.txt: some raw text' },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+  });
+
+  it('should read a .json file', async () => {
+    const mockJsonContent = JSON.stringify([
+      { name: 'You are a helpful assistant', role: 'system' },
+      { name: 'How do I get to the moon?', role: 'user' },
+    ]);
+
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(mockJsonContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const filePath = 'file://path/to/mock.json';
+    await expect(readPrompts([filePath])).resolves.toEqual([
+      {
+        raw: mockJsonContent,
+        label: expect.stringContaining(`mock.json: ${mockJsonContent}`),
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('mock.json'), 'utf8');
+    expect(fs.statSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should read a .jsonl file', async () => {
+    const data = [
+      [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'Who won the world series in {{ year }}?' },
+      ],
+      [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'Who won the superbowl in {{ year }}?' },
+      ],
+    ];
+
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(data.map((o) => JSON.stringify(o)).join('\n'));
+    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
+    await expect(readPrompts(['prompts.jsonl'])).resolves.toEqual([
+      {
+        raw: JSON.stringify(data[0]),
+        label: `prompts.jsonl: ${JSON.stringify(data[0])}`,
+      },
+      {
+        raw: JSON.stringify(data[1]),
+        label: `prompts.jsonl: ${JSON.stringify(data[1])}`,
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+  });
+
+  const yamlContent = dedent`
+    - role: user
+      content:
+        - type: text
+          text: "What's in this image?"
+        - type: image_url
+          image_url:
+            url: "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"`;
+
+  it('should read a .yaml file', async () => {
+    const expectedJson = JSON.stringify([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: "What's in this image?" },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
+            },
+          },
+        ],
+      },
+    ]);
+
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    await expect(readPrompts('prompts.yaml')).resolves.toEqual([
+      {
+        raw: expectedJson,
+        label: expect.stringContaining(
+          'prompts.yaml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}',
+        ),
+        config: undefined,
+      },
+    ]);
+  });
+
+  it('should read a .yml file', async () => {
+    const expectedJson = JSON.stringify([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: "What's in this image?" },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg',
+            },
+          },
+        ],
+      },
+    ]);
+
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(yamlContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    await expect(readPrompts('image-summary.yml')).resolves.toEqual([
+      {
+        raw: expectedJson,
+        label: expect.stringContaining(
+          'image-summary.yml: [{"role":"user","content":[{"type":"text","text":"What\'s in this image?"}',
+        ),
+        config: undefined,
+      },
+    ]);
+  });
+
+  it('should read a markdown file', async () => {
+    const mdContent = '# Test Heading\n\nThis is a test markdown file.';
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(mdContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+
+    await expect(readPrompts('test.md')).resolves.toEqual([
+      {
+        raw: mdContent,
+        label: 'test.md: # Test Heading\n\nThis is a test markdown file....',
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+    expect(fs.readFileSync).toHaveBeenCalledWith('test.md', 'utf8');
+  });
+
+  it('should read a Jinja2 file', async () => {
+    const jinjaContent =
+      'You are a helpful assistant.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(jinjaContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+
+    const result = await readPrompts('template.j2');
+
+    // Check that we get a result with the right content
+    expect(result).toHaveLength(1);
+    expect(result[0].raw).toEqual(jinjaContent);
+    expect(result[0].config).toBeUndefined();
+
+    // Check that the label contains the expected text but don't test exact truncation
+    expect(result[0].label).toContain('template.j2: You are a helpful assistant.');
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+    expect(fs.readFileSync).toHaveBeenCalledWith('template.j2', 'utf8');
+  });
+
+  it('should read a .py prompt object array', async () => {
+    const prompts = [
+      { id: 'prompts.py:prompt1', label: 'First prompt' },
+      { id: 'prompts.py:prompt2', label: 'Second prompt' },
+    ];
+
+    const code = dedent`
+      def prompt1:
+        return 'First prompt'
+      def prompt2:
+        return 'Second prompt'
+      `;
+    vi.mocked(fs.readFileSync).mockReturnValue(code);
+    await expect(readPrompts(prompts)).resolves.toEqual([
+      {
+        raw: code,
+        label: 'First prompt',
+        function: expect.any(Function),
+      },
+      {
+        raw: code,
+        label: 'Second prompt',
+        function: expect.any(Function),
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+  });
+
+  it('should read a .py file', async () => {
+    const code = `print('dummy prompt')`;
+    vi.mocked(fs.readFileSync).mockReturnValue(code);
+    await expect(readPrompts('prompt.py')).resolves.toEqual([
+      {
+        function: expect.any(Function),
+        label: `prompt.py: ${code}`,
+        raw: code,
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should read a .js file without named function', async () => {
+    const promptPath = 'prompt.js';
+    const mockFunction = () => console.log('dummy prompt');
+
+    vi.mocked(importModule).mockResolvedValueOnce(mockFunction);
+    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
+
+    await expect(readPrompts(promptPath)).resolves.toEqual([
+      {
+        raw: expect.stringMatching(/\(\)\s*=>\s*console\.log\(['"]dummy prompt['"]\)/),
+        label: 'prompt.js',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(importModule).toHaveBeenCalledWith(promptPath, undefined);
+    expect(fs.statSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should read a .js file with named function', async () => {
+    const promptPath = 'prompt.js:functionName';
+    const mockFunction = (_context: {
+      vars: Record<string, string | object>;
+      provider?: ApiProvider;
+    }) => 'dummy prompt result';
+
+    vi.mocked(importModule).mockResolvedValueOnce(mockFunction);
+    vi.mocked(fs.statSync).mockReturnValue({ isDirectory: () => false } as fs.Stats);
+
+    const result = await readPrompts(promptPath);
+    expect(result).toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'prompt.js:functionName',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+
+    const promptFunction = result[0].function as unknown as (context: {
+      vars: Record<string, string | object>;
+      provider?: ApiProvider;
+    }) => string;
+    expect(promptFunction({ vars: {}, provider: { id: () => 'foo' } as ApiProvider })).toBe(
+      'dummy prompt result',
+    );
+
+    expect(importModule).toHaveBeenCalledWith('prompt.js', 'functionName');
+    expect(fs.statSync).toHaveBeenCalledTimes(1);
+  });
+
+  it('should read a directory', async () => {
+    vi.mocked(fs.statSync).mockImplementation((filePath) => {
+      if (filePath.toString().endsWith('prompt1.txt')) {
+        return { isDirectory: () => false } as fs.Stats;
+      } else if (filePath.toString().endsWith('prompts')) {
+        return { isDirectory: () => true } as fs.Stats;
+      }
+      throw new Error(`Unexpected file path in test: ${filePath}`);
+    });
+    vi.mocked(globSync).mockImplementation(() => ['prompt1.txt', 'prompt2.txt']);
+    // The mocked paths here are an artifact of our globSync mock. In a real
+    // world setting we would get back `prompts/prompt1.txt` instead of `prompts/*/prompt1.txt`
+    // but for the sake of this test we are just going to pretend that the globSync
+    // mock is doing the right thing and giving us back the right paths.
+    vi.mocked(fs.readFileSync).mockImplementation((filePath) => {
+      if (
+        filePath.toString().endsWith('prompt1.txt') ||
+        filePath.toString().endsWith('*/prompt1.txt')
+      ) {
+        return 'Test prompt 1\n---\nTest prompt 2';
+      } else if (
+        filePath.toString().endsWith('prompt2.txt') ||
+        filePath.toString().endsWith('*/prompt2.txt')
+      ) {
+        return 'Test prompt 3\n---\nTest prompt 4\n---\nTest prompt 5';
+      }
+      throw new Error(`Unexpected file path in test: ${filePath}`);
+    });
+    await expect(readPrompts(['prompts/*'])).resolves.toEqual([
+      {
+        label: expect.stringMatching('prompt1.txt: Test prompt 1'),
+        raw: 'Test prompt 1',
+      },
+      {
+        label: expect.stringMatching('prompt1.txt: Test prompt 2'),
+        raw: 'Test prompt 2',
+      },
+      {
+        label: expect.stringMatching('prompt2.txt: Test prompt 3'),
+        raw: 'Test prompt 3',
+      },
+      {
+        label: expect.stringMatching('prompt2.txt: Test prompt 4'),
+        raw: 'Test prompt 4',
+      },
+      {
+        label: expect.stringMatching('prompt2.txt: Test prompt 5'),
+        raw: 'Test prompt 5',
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+    expect(fs.statSync).toHaveBeenCalledTimes(3);
+  });
+
+  it('should fall back to a string if maybeFilePath is true but a file does not exist', async () => {
+    vi.mocked(globSync).mockReturnValueOnce([]);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+    await expect(readPrompts('non-existent-file.txt*')).resolves.toEqual([
+      { raw: 'non-existent-file.txt*', label: 'non-existent-file.txt*' },
+    ]);
+  });
+
+  it('should handle a prompt with a function', async () => {
+    const promptWithFunction: Partial<Prompt> = {
+      raw: 'dummy raw text',
+      label: 'Function Prompt',
+      function: vi.fn().mockResolvedValue('Hello, world!'),
+    };
+
+    await expect(readPrompts([promptWithFunction])).resolves.toEqual([
+      {
+        raw: 'dummy raw text',
+        label: 'Function Prompt',
+        function: expect.any(Function),
+      },
+    ]);
+    expect(promptWithFunction.function).not.toHaveBeenCalled();
+  });
+  it('should read a single-column CSV file with header', async () => {
+    const csvContent = dedent`
+      prompt
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await readPrompts(['test.csv']);
+
+    expect(result).toHaveLength(3);
+    expect(result[0]).toEqual({
+      raw: 'Tell me about {{topic}}',
+      label: 'Prompt 1 - Tell me about {{topic}}',
+    });
+    expect(result[1]).toEqual({
+      raw: 'Explain {{topic}} in simple terms',
+      label: 'Prompt 2 - Explain {{topic}} in simple terms',
+    });
+    expect(result[2]).toEqual({
+      raw: 'Write a poem about {{topic}}',
+      label: 'Prompt 3 - Write a poem about {{topic}}',
+    });
+  });
+
+  it('should read a single-column CSV file without header', async () => {
+    const csvContent = dedent`
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await readPrompts(['test.csv']);
+
+    expect(result).toHaveLength(3);
+    expect(result[0]).toEqual({
+      raw: 'Tell me about {{topic}}',
+      label: 'Prompt 1 - Tell me about {{topic}}',
+    });
+    expect(result[1]).toEqual({
+      raw: 'Explain {{topic}} in simple terms',
+      label: 'Prompt 2 - Explain {{topic}} in simple terms',
+    });
+    expect(result[2]).toEqual({
+      raw: 'Write a poem about {{topic}}',
+      label: 'Prompt 3 - Write a poem about {{topic}}',
+    });
+  });
+
+  it('should read a two-column CSV file with prompt and label', async () => {
+    const csvContent = dedent`
+      prompt,label
+      Tell me about {{topic}},Basic Query
+      Explain {{topic}} in simple terms,Simple Explanation
+      Write a poem about {{topic}},Poetry Generator
+    `;
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await readPrompts(['test.csv']);
+
+    expect(result).toHaveLength(3);
+    expect(result[0]).toEqual({
+      raw: 'Tell me about {{topic}}',
+      label: 'Basic Query',
+    });
+    expect(result[1]).toEqual({
+      raw: 'Explain {{topic}} in simple terms',
+      label: 'Simple Explanation',
+    });
+    expect(result[2]).toEqual({
+      raw: 'Write a poem about {{topic}}',
+      label: 'Poetry Generator',
+    });
+  });
+});
+
+describe('readProviderPromptMap', () => {
+  let config: Pick<Partial<EvaluateTestSuite>, 'providers'>;
+  let parsedPrompts: Prompt[];
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    parsedPrompts = [
+      { label: 'prompt1', raw: 'prompt1' },
+      { label: 'prompt2', raw: 'prompt2' },
+    ];
+  });
+
+  it('should return an empty object if config.providers is undefined', () => {
+    config = {};
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({});
+  });
+
+  it('should return a map with all prompts if config.providers is a string', () => {
+    config = { providers: 'provider1' };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should return a map with all prompts if config.providers is a function', () => {
+    config = {
+      providers: () =>
+        Promise.resolve({
+          providerName: 'Custom function',
+          prompts: ['prompt1', 'prompt2'],
+        }) as Promise<ProviderResponse>,
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      'Custom function': ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should handle provider objects with id and prompts', () => {
+    config = {
+      providers: [{ id: 'provider1', prompts: ['customPrompt1'] }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
+  });
+
+  it('should handle a top-level ApiProvider object', () => {
+    const provider: ApiProvider = {
+      id: () => 'provider1',
+      callApi: vi.fn<ApiProvider['callApi']>(),
+      label: 'providerLabel',
+    };
+    config = { providers: provider };
+
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should handle ApiProvider objects in provider arrays', () => {
+    const provider: ApiProvider = {
+      id: () => 'provider1',
+      callApi: vi.fn<ApiProvider['callApi']>(),
+      label: 'providerLabel',
+    };
+    config = { providers: [provider] };
+
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['prompt1', 'prompt2'],
+      providerLabel: ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should handle provider objects with id, label, and prompts', () => {
+    config = {
+      providers: [{ id: 'provider1', label: 'providerLabel', prompts: ['customPrompt1'] }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['customPrompt1'],
+      providerLabel: ['customPrompt1'],
+    });
+  });
+
+  it('should handle provider options map with id and prompts', () => {
+    config = {
+      providers: [
+        {
+          originalProvider: {
+            id: 'provider1',
+            prompts: ['customPrompt1'],
+          },
+        },
+      ],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
+  });
+
+  it('should handle provider options map without id and use original id', () => {
+    config = {
+      providers: [
+        {
+          originalProvider: {
+            prompts: ['customPrompt1'],
+          },
+        },
+      ],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      originalProvider: ['customPrompt1'],
+    });
+  });
+
+  it('should use rawProvider.prompts if provided for provider objects with id', () => {
+    config = {
+      providers: [{ id: 'provider1', prompts: ['customPrompt1'] }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ provider1: ['customPrompt1'] });
+  });
+
+  it('should fall back to allPrompts if no prompts provided for provider objects with id', () => {
+    config = {
+      providers: [{ id: 'provider1' }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should use rawProvider.prompts for both id and label if provided', () => {
+    config = {
+      providers: [{ id: 'provider1', label: 'providerLabel', prompts: ['customPrompt1'] }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['customPrompt1'],
+      providerLabel: ['customPrompt1'],
+    });
+  });
+
+  it('should fall back to allPrompts for both id and label if no prompts provided', () => {
+    config = {
+      providers: [{ id: 'provider1', label: 'providerLabel' }],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      provider1: ['prompt1', 'prompt2'],
+      providerLabel: ['prompt1', 'prompt2'],
+    });
+  });
+
+  it('should use providerObject.id from ProviderOptionsMap when provided', () => {
+    config = {
+      providers: [
+        {
+          originalProvider: {
+            id: 'explicitId',
+            prompts: ['customPrompt1'],
+          },
+        },
+      ],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({ explicitId: ['customPrompt1'] });
+  });
+
+  it('should fallback to originalId when providerObject.id is not specified in ProviderOptionsMap', () => {
+    config = {
+      providers: [
+        {
+          originalProvider: {
+            // 'originalProvider' is treated as originalId
+            prompts: ['customPrompt1'],
+          },
+        },
+      ],
+    };
+    expect(readProviderPromptMap(config, parsedPrompts)).toEqual({
+      originalProvider: ['customPrompt1'],
+    });
+  });
+});
+
+describe('processPrompts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process function prompts', async () => {
+    function testFunction(vars: any) {
+      return `Hello ${vars.name}`;
+    }
+
+    const result = await processPrompts([testFunction]);
+
+    expect(result).toEqual([
+      {
+        raw: testFunction.toString(),
+        label: 'testFunction',
+        function: testFunction,
+      },
+    ]);
+  });
+
+  it('should process function prompts with no name', async () => {
+    function promptFn(vars: any) {
+      return `Hello ${vars.name}`;
+    }
+    Object.defineProperty(promptFn, 'name', { value: '' });
+
+    const result = await processPrompts([promptFn]);
+
+    expect(result).toEqual([
+      {
+        raw: promptFn.toString(),
+        label: '',
+        function: expect.any(Function),
+      },
+    ]);
+    expect(fs.readFileSync).toHaveBeenCalledTimes(0);
+  });
+
+  it('should process string prompts by calling readPrompts', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValueOnce('test prompt');
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await processPrompts(['test.txt']);
+
+    expect(result).toEqual([
+      {
+        raw: 'test prompt',
+        label: 'test.txt: test prompt',
+      },
+    ]);
+  });
+
+  it('should process Jinja2 files', async () => {
+    const jinjaContent =
+      'You are a helpful assistant for {{ user }}.\nPlease provide information about {{ topic }}.';
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(jinjaContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+    vi.mocked(maybeFilePath).mockReturnValueOnce(true);
+
+    const result = await processPrompts(['template.j2']);
+
+    // Check that we get a result with the right content
+    expect(result).toHaveLength(1);
+    expect(result[0].raw).toEqual(jinjaContent);
+    expect(result[0].config).toBeUndefined();
+
+    // Check that the label contains the expected text but don't test exact truncation
+    expect(result[0].label).toContain('template.j2: You are a helpful assistant for {{ user }}.');
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+    expect(fs.readFileSync).toHaveBeenCalledWith('template.j2', 'utf8');
+  });
+
+  it('should process valid prompt schema objects', async () => {
+    const validPrompt = {
+      raw: 'test prompt',
+      label: 'test label',
+    };
+
+    const result = await processPrompts([validPrompt]);
+
+    expect(result).toEqual([validPrompt]);
+  });
+
+  it('should fall back to JSON serialization for invalid prompt schema objects', async () => {
+    const invalidPrompt = {
+      invalidField: 'some value',
+      anotherField: 123,
+    };
+
+    const result = await processPrompts([invalidPrompt]);
+
+    expect(result).toEqual([
+      {
+        raw: JSON.stringify(invalidPrompt),
+        label: JSON.stringify(invalidPrompt),
+      },
+    ]);
+  });
+
+  it('should process multiple prompts of different types', async () => {
+    function testFunction(vars: any) {
+      return `Hello ${vars.name}`;
+    }
+
+    const validPrompt = {
+      raw: 'test prompt',
+      label: 'test label',
+    };
+
+    vi.mocked(fs.readFileSync).mockReturnValueOnce('file prompt');
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await processPrompts([testFunction, 'test.txt', validPrompt]);
+
+    expect(result).toEqual([
+      {
+        raw: testFunction.toString(),
+        label: 'testFunction',
+        function: testFunction,
+      },
+      {
+        raw: 'file prompt',
+        label: 'test.txt: file prompt',
+      },
+      validPrompt,
+    ]);
+  });
+
+  it('should process CSV files', async () => {
+    const csvContent = `prompt,label
+Tell me about {{topic}},Basic Query
+Explain {{topic}} in simple terms,Simple Explanation`;
+    vi.mocked(fs.readFileSync).mockReturnValueOnce(csvContent);
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await processPrompts(['test.csv']);
+
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Basic Query',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Simple Explanation',
+      },
+    ]);
+  });
+
+  it('should flatten array results from readPrompts', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValueOnce('prompt1\n---\nprompt2');
+    vi.mocked(fs.statSync).mockReturnValueOnce({ isDirectory: () => false } as fs.Stats);
+
+    const result = await processPrompts(['test.txt']);
+
+    expect(result).toEqual([
+      { raw: 'prompt1', label: 'test.txt: prompt1' },
+      { raw: 'prompt2', label: 'test.txt: prompt2' },
+    ]);
+    expect(Array.isArray(result)).toBe(true);
+    expect(result).toHaveLength(2);
+  });
+});

test/prompts/processors/csv.test.ts

@@ -0,0 +1,242 @@
+import fs from 'fs';
+
+import dedent from 'ts-dedent';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processCsvPrompts } from '../../../src/prompts/processors/csv';
+import { mockProcessEnv } from '../../util/utils';
+
+import type { Prompt } from '../../../src/types/index';
+
+vi.mock('fs');
+
+describe('processCsvPrompts', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should process a single column CSV with header', async () => {
+    const csvContent = dedent`
+      prompt
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Prompt 1 - Tell me about {{topic}}',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Prompt 2 - Explain {{topic}} in simple terms',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Prompt 3 - Write a poem about {{topic}}',
+      },
+    ]);
+  });
+
+  it('should process a single column text file without header', async () => {
+    const csvContent = dedent`
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Prompt 1 - Tell me about {{topic}}',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Prompt 2 - Explain {{topic}} in simple terms',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Prompt 3 - Write a poem about {{topic}}',
+      },
+    ]);
+  });
+
+  it('should process a two column CSV with prompt and label', async () => {
+    const csvContent = dedent`
+      prompt,label
+      Tell me about {{topic}},Basic Query
+      Explain {{topic}} in simple terms,Simple Explanation
+      Write a poem about {{topic}},Poetry Generator
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Basic Query',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Simple Explanation',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Poetry Generator',
+      },
+    ]);
+  });
+
+  it('should generate labels from prompt content when not provided', async () => {
+    const csvContent = dedent`
+      prompt
+      This is a very long prompt that should be truncated for the label
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(1);
+    expect(result).toEqual([
+      {
+        raw: 'This is a very long prompt that should be truncated for the label',
+        label: 'Prompt 1 - This is a very long prompt that should be truncated for the label',
+      },
+    ]);
+  });
+
+  it('should use base prompt properties if provided', async () => {
+    const csvContent = dedent`
+      prompt
+      Tell me about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const basePrompt: Partial<Prompt> = {
+      label: 'Base Label',
+      id: 'base-id',
+    };
+
+    const result = await processCsvPrompts('prompts.csv', basePrompt);
+
+    expect(result).toHaveLength(1);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Base Label',
+        id: 'base-id',
+      },
+    ]);
+  });
+
+  it('should skip rows with missing prompt values', async () => {
+    const csvContent = dedent`
+      prompt,label
+      Tell me about {{topic}},Basic Query
+
+      Write a poem about {{topic}},Poetry Generator
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(2);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Basic Query',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Poetry Generator',
+      },
+    ]);
+  });
+
+  it('should handle custom delimiters', async () => {
+    const csvContent = dedent`
+      prompt;label
+      Tell me about {{topic}};Basic Query
+      Explain {{topic}} in simple terms;Simple Explanation
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+    const restoreEnv = mockProcessEnv({ PROMPTFOO_CSV_DELIMITER: ';' });
+    try {
+      const result = await processCsvPrompts('prompts.csv', {});
+
+      expect(result).toHaveLength(2);
+      expect(result).toEqual([
+        {
+          raw: 'Tell me about {{topic}}',
+          label: 'Basic Query',
+        },
+        {
+          raw: 'Explain {{topic}} in simple terms',
+          label: 'Simple Explanation',
+        },
+      ]);
+    } finally {
+      restoreEnv();
+    }
+  });
+
+  it('should handle empty files', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValue('');
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle malformed CSV by falling back to line-by-line processing', async () => {
+    const csvContent = dedent`
+      "prompt","label"
+      "Tell me about {{topic}},"Basic Query"
+      "Malformed line with unbalanced quotes
+      "Another line
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(4);
+    expect(result[0].raw).toBe('"prompt","label"');
+    expect(result[1].raw).toBe('"Tell me about {{topic}},"Basic Query"');
+    expect(result[2].raw).toBe('"Malformed line with unbalanced quotes');
+    expect(result[3].raw).toBe('"Another line');
+  });
+
+  it('should handle malformed CSV with a header row correctly', async () => {
+    const csvContent = dedent`
+      prompt
+      "Malformed CSV with a quote problem
+      Another prompt line
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(2);
+    expect(result[0].raw).toBe('"Malformed CSV with a quote problem');
+    expect(result[1].raw).toBe('Another prompt line');
+  });
+});

test/prompts/processors/executable.test.ts

@@ -0,0 +1,199 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { processExecutableFile } from '../../../src/prompts/processors/executable';
+
+import type { ApiProvider } from '../../../src/types/index';
+
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+vi.mock('../../../src/cache', () => ({
+  getCache: vi.fn(() => ({
+    get: vi.fn(),
+    set: vi.fn(),
+  })),
+  isCacheEnabled: vi.fn(() => false),
+}));
+
+describe('processExecutableFile', () => {
+  const mockProvider = {
+    id: vi.fn(() => 'test-provider'),
+    label: 'Test Provider',
+    callApi: vi.fn(),
+  } as ApiProvider;
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  const describeUnix = process.platform === 'win32' ? describe.skip : describe;
+
+  // Cross-platform tests
+  it('should process a script with exec: prefix', async () => {
+    const scriptPath =
+      process.platform === 'win32' ? 'cmd.exe /c echo "test"' : '/usr/bin/echo "test"';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    expect(prompts[0].raw).toBe(scriptPath);
+    expect(typeof prompts[0].function).toBe('function');
+  });
+
+  it('should use custom label when provided', async () => {
+    const scriptPath = process.platform === 'win32' ? 'cmd.exe' : '/bin/echo';
+    const prompts = await processExecutableFile(scriptPath, { label: 'Custom Label' });
+
+    expect(prompts[0].label).toBe('Custom Label');
+  });
+
+  it('should handle binary executables', async () => {
+    const scriptPath =
+      process.platform === 'win32' ? 'C:\\Windows\\System32\\cmd.exe' : '/usr/bin/ls';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    // Binary files should not be read as raw content
+    expect(prompts[0].raw).toBe(scriptPath);
+  });
+
+  it('should handle non-existent files gracefully', async () => {
+    const scriptPath = '/non/existent/script.sh';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    expect(prompts[0].raw).toBe(scriptPath);
+  });
+
+  // Unix-specific tests
+  describeUnix('Unix shell script tests', () => {
+    let sharedPrompts: Awaited<ReturnType<typeof processExecutableFile>>;
+    let tempDir: string;
+    let scriptPath: string;
+
+    beforeAll(async () => {
+      tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-executable-test-'));
+      scriptPath = path.join(tempDir, 'shared-test-script.sh');
+      fs.writeFileSync(
+        scriptPath,
+        `#!/bin/sh
+context="$1"
+case "$context" in
+  *'"mode":"args"'*)
+    printf '%s\\n' "Context received: $context"
+    ;;
+  *'"mode":"config"'*)
+    printf '%s\\n' "Test output"
+    ;;
+  *'"mode":"stderr"'*)
+    printf '%s\\n' "Error message" >&2
+    printf '%s\\n' "Normal output"
+    ;;
+  *'"mode":"error"'*)
+    printf '%s\\n' "Error only" >&2
+    exit 1
+    ;;
+  *'"mode":"relative"'*)
+    printf '%s\\n' "Relative path works"
+    ;;
+  *)
+    printf '%s\\n' "Hello from shell script"
+    ;;
+esac
+`,
+      );
+      fs.chmodSync(scriptPath, 0o755);
+      sharedPrompts = await processExecutableFile(scriptPath, {});
+    });
+
+    afterAll(() => {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    });
+
+    it('should process a simple shell script', async () => {
+      expect(sharedPrompts).toHaveLength(1);
+      expect(sharedPrompts[0].label).toBe(scriptPath);
+      expect(sharedPrompts[0].raw).toContain('#!/bin/sh');
+      expect(typeof sharedPrompts[0].function).toBe('function');
+
+      const result = await sharedPrompts[0].function!({
+        vars: { test: 'value' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Hello from shell script');
+    });
+
+    it('should handle scripts with arguments', async () => {
+      const result = await sharedPrompts[0].function!({
+        vars: { mode: 'args', name: 'test' },
+        provider: mockProvider,
+      });
+
+      // The script should receive the context as JSON
+      expect(result).toContain('Context received:');
+      expect(result).toContain('"vars"');
+      expect(result).toContain('"name":"test"');
+    });
+
+    it('should pass config to the function', async () => {
+      const config = { temperature: 0.5 };
+      const prompts = await processExecutableFile(scriptPath, { config });
+
+      expect(prompts[0].config).toEqual(config);
+
+      const result = await prompts[0].function!({
+        vars: { mode: 'config' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Test output');
+    });
+
+    it('should handle scripts that output to stderr', async () => {
+      const result = await sharedPrompts[0].function!({
+        vars: { mode: 'stderr' },
+        provider: mockProvider,
+      });
+
+      // Should return stdout even if there's stderr
+      expect(result).toBe('Normal output');
+    });
+
+    it('should reject when script fails with no stdout', async () => {
+      await expect(
+        sharedPrompts[0].function!({
+          vars: { mode: 'error' },
+          provider: mockProvider,
+        }),
+      ).rejects.toThrow();
+    });
+
+    it('should resolve relative paths from prompt.config.basePath', async () => {
+      const relativeScriptPath = `.${path.sep}${path.basename(scriptPath)}`;
+      const prompts = await processExecutableFile(relativeScriptPath, {
+        config: { basePath: tempDir },
+      });
+
+      expect(prompts).toHaveLength(1);
+      expect(prompts[0].label).toBe(relativeScriptPath);
+      expect(typeof prompts[0].function).toBe('function');
+
+      const result = await prompts[0].function!({
+        vars: { mode: 'relative' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Relative path works');
+    });
+  });
+});

test/prompts/processors/javascript.test.ts

@@ -0,0 +1,157 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { importModule } from '../../../src/esm';
+import { processJsFile, transformContext } from '../../../src/prompts/processors/javascript';
+import invariant from '../../../src/util/invariant';
+
+vi.mock('../../../src/esm', () => ({
+  importModule: vi.fn(),
+}));
+
+describe('transformContext', () => {
+  it('should transform context with provider and config', () => {
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel', callApi: vi.fn() },
+      config: { configKey: 'configValue' },
+    };
+    const result = transformContext(context);
+    expect(result).toEqual({
+      vars: { key: 'value' },
+      provider: { id: 'providerId', label: 'providerLabel' },
+      config: { configKey: 'configValue' },
+    });
+  });
+
+  it('should transform context with provider and empty config', () => {
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel', callApi: vi.fn() },
+    };
+    const result = transformContext(context);
+    expect(result).toEqual({
+      vars: { key: 'value' },
+      provider: { id: 'providerId', label: 'providerLabel' },
+      config: {},
+    });
+  });
+
+  it('should throw an error if provider is missing', () => {
+    const context = { vars: { key: 'value' } };
+    expect(() => transformContext(context)).toThrow('Provider is required');
+  });
+});
+
+describe('processJsFile', () => {
+  const mockImportModule = vi.mocked(importModule);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid JavaScript file without a function name or a label', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, {}, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with a label without a function name', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, { label: 'myLabel' }, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'myLabel',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with a function name without a label', async () => {
+    const filePath = 'file.js';
+    const functionName = 'myFunction';
+    const mockFunction = () => 'dummy prompt';
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, {}, functionName)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js:myFunction',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, functionName);
+  });
+
+  it('should process a valid JavaScript file with a label and a function name', async () => {
+    const filePath = 'file.js';
+    const functionName = 'myFunction';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, { label: 'myLabel' }, functionName)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'myLabel',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, functionName);
+  });
+
+  it('should throw an error if the file cannot be imported', async () => {
+    const filePath = 'nonexistent.js';
+    mockImportModule.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    await expect(processJsFile(filePath, {}, undefined)).rejects.toThrow('File not found');
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with config', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    const config = { key: 'value' };
+    await expect(processJsFile(filePath, { config }, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js',
+        function: expect.any(Function),
+        config: { key: 'value' },
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should pass config to the prompt function', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    const config = { key: 'value' };
+    const result = await processJsFile(filePath, { config }, undefined);
+    const promptFunction = result[0].function;
+    invariant(promptFunction, 'Prompt function is required');
+    await promptFunction({
+      vars: {},
+      provider: { id: () => 'test', label: 'Test', callApi: vi.fn() },
+    });
+    expect(mockFunction).toHaveBeenCalledWith(
+      expect.objectContaining({
+        config: { key: 'value' },
+      }),
+    );
+  });
+});

test/prompts/processors/jinja.test.ts

@@ -0,0 +1,102 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJinjaFile } from '../../../src/prompts/processors/jinja';
+
+vi.mock('fs');
+
+describe('processJinjaFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a Jinja2 file without a label', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, {});
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a Jinja2 file with a label', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, { label: 'Custom Label' });
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: 'Custom Label',
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should include config when provided', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    const config = { temperature: 0.7, max_tokens: 150 };
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, { config });
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.j2';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processJinjaFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should handle variable interpolation syntax properly', () => {
+    const filePath = 'complex.j2';
+    const fileContent = `
+    {% if condition %}
+      Handle {{ variable1 }} with condition
+    {% else %}
+      Handle {{ variable2 }} without condition
+    {% endif %}
+    `;
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, {});
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+});

test/prompts/processors/json.test.ts

@@ -0,0 +1,155 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJsonFile } from '../../../src/prompts/processors/json';
+import * as fileModule from '../../../src/util/file';
+
+vi.mock('fs');
+vi.mock('../../../src/util/file');
+
+describe('processJsonFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+  const mockMaybeLoadConfigFromExternalFile = vi.mocked(fileModule.maybeLoadConfigFromExternalFile);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // By default, maybeLoadConfigFromExternalFile returns its input unchanged
+    mockMaybeLoadConfigFromExternalFile.mockImplementation((input) => input);
+  });
+
+  it('should process a valid JSON file without a label', () => {
+    const filePath = 'file.json';
+    const fileContent = JSON.stringify({ key: 'value' });
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent}`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid JSON file with a label', () => {
+    const filePath = 'file.json';
+    const fileContent = JSON.stringify({ key: 'value' });
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: fileContent,
+        label: `Label`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.json';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processJsonFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  describe('recursive file:// resolution', () => {
+    it('should recursively resolve file:// references in JSON content', () => {
+      const filePath = 'conversation.json';
+      const fileContent = JSON.stringify([
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ]);
+
+      const parsedJson = [
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ];
+
+      const resolvedContent = [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'What is 2 + 2?' },
+      ];
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle nested file:// references in complex JSON structures', () => {
+      const filePath = 'config.json';
+      const fileContent = JSON.stringify({
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.json',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      });
+
+      const parsedJson = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.json',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      };
+
+      const resolvedContent = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 0.7,
+            tools: [{ name: 'search', description: 'Search the web' }],
+          },
+        },
+        messages: [{ role: 'system', content: 'You are a helpful assistant.' }],
+      };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle invalid JSON and return original content', () => {
+      const filePath = 'invalid.json';
+      const fileContent = 'This is not valid JSON {]';
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processJsonFile(filePath, {});
+
+      // When JSON parsing fails, it should return the original content
+      expect(mockMaybeLoadConfigFromExternalFile).not.toHaveBeenCalled();
+      expect(result[0].raw).toBe(fileContent);
+    });
+
+    it('should handle when maybeLoadConfigFromExternalFile returns unchanged content', () => {
+      const filePath = 'no-refs.json';
+      const fileContent = JSON.stringify({ key1: 'value1', key2: 'value2' });
+
+      const parsedJson = { key1: 'value1', key2: 'value2' };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(parsedJson);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(parsedJson));
+    });
+  });
+});

test/prompts/processors/jsonl.test.ts

@@ -0,0 +1,83 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJsonlFile } from '../../../src/prompts/processors/jsonl';
+
+vi.mock('fs');
+
+describe('processJsonlFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid JSONL file without a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}]\n[{"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, {})).toEqual([
+      {
+        raw: '[{"key1": "value1"}]',
+        label: 'file.jsonl: [{"key1": "value1"}]',
+      },
+      {
+        raw: '[{"key2": "value2"}]',
+        label: 'file.jsonl: [{"key2": "value2"}]',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with a single record without a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}, {"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, {})).toEqual([
+      {
+        raw: '[{"key1": "value1"}, {"key2": "value2"}]',
+        label: `file.jsonl`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with a single record and a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}, {"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: '[{"key1": "value1"}, {"key2": "value2"}]',
+        label: `Label`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with multiple records and a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}]\n[{"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: '[{"key1": "value1"}]',
+        label: `Label: [{"key1": "value1"}]`,
+      },
+      {
+        raw: '[{"key2": "value2"}]',
+        label: `Label: [{"key2": "value2"}]`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.jsonl';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+    expect(() => processJsonlFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+});

test/prompts/processors/markdown.test.ts

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processMarkdownFile } from '../../../src/prompts/processors/markdown';
+
+vi.mock('fs');
+
+describe('processMarkdownFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid Markdown file without a label', () => {
+    const filePath = 'file.md';
+    const fileContent = '# Heading\n\nThis is some markdown content.';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: # Heading\n\nThis is some markdown content....`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid Markdown file with a label', () => {
+    const filePath = 'file.md';
+    const fileContent = '# Heading\n\nThis is some markdown content.';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, { label: 'Custom Label' })).toEqual([
+      {
+        raw: fileContent,
+        label: 'Custom Label',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should truncate the label for long Markdown files', () => {
+    const filePath = 'file.md';
+    const fileContent = '# ' + 'A'.repeat(100);
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: # ${'A'.repeat(48)}...`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.md';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processMarkdownFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+});

test/prompts/processors/python.test.ts

@@ -0,0 +1,99 @@
+import * as fs from 'fs';
+
+import dedent from 'dedent';
+import { describe, expect, it, vi } from 'vitest';
+import {
+  processPythonFile,
+  pythonPromptFunction,
+  pythonPromptFunctionLegacy,
+} from '../../../src/prompts/processors/python';
+
+vi.mock('fs');
+vi.mock('../../../src/prompts/processors/python', async () => {
+  const actual = await vi.importActual('../../../src/prompts/processors/python');
+  return {
+    ...actual,
+    pythonPromptFunction: vi.fn(),
+    pythonPromptFunctionLegacy: vi.fn(),
+  };
+});
+
+describe('processPythonFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  it('should process a valid Python file without a function name or label', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, {}, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `${filePath}: ${fileContent}`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a function name without a label', () => {
+    const filePath = 'file.py';
+    const fileContent = dedent`
+    def testFunction(context):
+      print("Hello, world!")`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunction).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, {}, 'testFunction')).toEqual([
+      {
+        function: expect.any(Function),
+        raw: fileContent,
+        label: `file.py:testFunction`,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a label without a function name', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, { label: 'Label' }, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `Label`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a label and function name', () => {
+    const filePath = 'file.py';
+    const fileContent = dedent`
+    def testFunction(context):
+      print("Hello, world!")`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunction).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, { label: 'Label' }, 'testFunction')).toEqual([
+      {
+        function: expect.any(Function),
+        label: `Label`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with config', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    const config = { key: 'value' };
+    expect(processPythonFile(filePath, { config }, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `${filePath}: ${fileContent}`,
+        raw: fileContent,
+        config: { key: 'value' },
+      },
+    ]);
+  });
+});

test/prompts/processors/python.utils.test.ts

@@ -0,0 +1,91 @@
+import { PythonShell } from 'python-shell';
+import { describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import {
+  pythonPromptFunction,
+  pythonPromptFunctionLegacy,
+} from '../../../src/prompts/processors/python';
+import { runPython } from '../../../src/python/pythonUtils';
+
+import type { ApiProvider } from '../../../src/types/index';
+
+vi.mock('fs');
+vi.mock('python-shell');
+vi.mock('../../../src/python/pythonUtils');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+  },
+}));
+
+describe('pythonPromptFunction', () => {
+  interface PythonContext {
+    vars: Record<string, string | object>;
+    provider: ApiProvider;
+  }
+
+  it('should call python wrapper function with correct arguments', async () => {
+    const filePath = 'path/to/script.py';
+    const functionName = 'testFunction';
+    const context = {
+      vars: { key: 'value' },
+      provider: {
+        id: () => 'providerId',
+        label: 'providerLabel',
+        callApi: vi.fn(),
+      } as ApiProvider,
+    } as PythonContext;
+
+    const mockRunPython = vi.mocked(runPython);
+    mockRunPython.mockResolvedValue('mocked result');
+
+    await expect(pythonPromptFunction(filePath, functionName, context)).resolves.toBe(
+      'mocked result',
+    );
+    expect(mockRunPython).toHaveBeenCalledWith(filePath, functionName, [
+      {
+        ...context,
+        provider: {
+          id: 'providerId',
+          label: 'providerLabel',
+        },
+        config: {},
+      },
+    ]);
+  });
+
+  it('should call legacy function with correct arguments', async () => {
+    const filePath = 'path/to/script.py';
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel' } as ApiProvider,
+    } as PythonContext;
+    const mockPythonShellRun = vi.mocked(PythonShell.run);
+    const mockLoggerDebug = vi.mocked(logger.debug);
+    mockPythonShellRun.mockImplementation(() => {
+      return Promise.resolve(['mocked result']);
+    });
+    await expect(pythonPromptFunctionLegacy(filePath, context)).resolves.toBe('mocked result');
+    expect(mockPythonShellRun).toHaveBeenCalledWith(filePath, {
+      mode: 'text',
+      pythonPath: process.env.PROMPTFOO_PYTHON || 'python',
+      args: [
+        JSON.stringify({
+          vars: context.vars,
+          provider: {
+            id: context.provider.id(),
+            label: context.provider.label,
+          },
+          config: {},
+        }),
+      ],
+    });
+    expect(mockLoggerDebug).toHaveBeenCalledWith(`Executing python prompt script ${filePath}`);
+    expect(mockLoggerDebug).toHaveBeenCalledWith(
+      `Python prompt script ${filePath} returned: mocked result`,
+    );
+  });
+});

test/prompts/processors/string.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest';
+import { processString } from '../../../src/prompts/processors/string';
+
+import type { Prompt } from '../../../src/types/index';
+
+describe('processString', () => {
+  it('should process a valid string prompt without a label', () => {
+    const prompt: Partial<Prompt> = { raw: 'This is a prompt' };
+    expect(processString(prompt)).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'This is a prompt',
+      },
+    ]);
+  });
+
+  it('should process a valid string prompt with a label', () => {
+    const prompt: Partial<Prompt> = { raw: 'This is a prompt', label: 'Label' };
+    expect(processString(prompt)).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'Label',
+      },
+    ]);
+  });
+});

test/prompts/processors/text.test.ts

@@ -0,0 +1,107 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { PROMPT_DELIMITER } from '../../../src/prompts/constants';
+import { processTxtFile } from '../../../src/prompts/processors/text';
+
+vi.mock('fs');
+
+describe('processTxtFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a text file with single prompt and no label', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'This is a prompt';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'file.txt: This is a prompt',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a text file with single prompt and a label', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'This is a prompt';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, { label: 'prompt 1' })).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'prompt 1: file.txt: This is a prompt',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a text file with multiple prompts and a label', () => {
+    const fileContent = `Prompt 1\n${PROMPT_DELIMITER}\nPrompt 2\n${PROMPT_DELIMITER}\nPrompt 3`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile('file.txt', { label: 'Label' })).toEqual([
+      {
+        raw: 'Prompt 1',
+        label: `Label: file.txt: Prompt 1`,
+      },
+      {
+        raw: 'Prompt 2',
+        label: `Label: file.txt: Prompt 2`,
+      },
+      {
+        raw: 'Prompt 3',
+        label: `Label: file.txt: Prompt 3`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith('file.txt', 'utf-8');
+  });
+
+  it('should handle text file with leading and trailing delimiters', () => {
+    const filePath = 'file.txt';
+    const fileContent = `${PROMPT_DELIMITER}\nPrompt 1\n${PROMPT_DELIMITER}\nPrompt 2\n${PROMPT_DELIMITER}`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'Prompt 1',
+        label: `${filePath}: Prompt 1`,
+      },
+      {
+        raw: 'Prompt 2',
+        label: `${filePath}: Prompt 2`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should return an empty array for a file with only delimiters', () => {
+    const filePath = 'file.txt';
+    const fileContent = `${PROMPT_DELIMITER}\n${PROMPT_DELIMITER}`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should return an empty array for an empty file', () => {
+    const filePath = 'file.txt';
+    const fileContent = '';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should not split on lines that contain repeated hyphens', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'Line 1\n-----------------------------------\nLine 2';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'Line 1\n-----------------------------------\nLine 2',
+        label: `${filePath}: Line 1\n-----------------------------------\nLine 2`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+});

test/prompts/processors/yaml.test.ts

@@ -0,0 +1,285 @@
+import * as fs from 'fs';
+
+import dedent from 'dedent';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import { processYamlFile } from '../../../src/prompts/processors/yaml';
+import * as fileModule from '../../../src/util/file';
+
+vi.mock('fs');
+vi.mock('../../../src/util/file');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+  },
+}));
+
+describe('processYamlFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+  const mockMaybeLoadConfigFromExternalFile = vi.mocked(fileModule.maybeLoadConfigFromExternalFile);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // By default, maybeLoadConfigFromExternalFile returns its input unchanged
+    mockMaybeLoadConfigFromExternalFile.mockImplementation((input) => input);
+  });
+
+  it('should process a valid YAML file without a label', () => {
+    const filePath = 'file.yaml';
+    const fileContent = 'key: value';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processYamlFile(filePath, {})).toEqual([
+      {
+        raw: JSON.stringify({ key: 'value' }),
+        label: `${filePath}: ${JSON.stringify({ key: 'value' })}`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid YAML file with a label', () => {
+    const filePath = 'file.yaml';
+    const fileContent = 'key: value';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processYamlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: JSON.stringify({ key: 'value' }),
+        label: 'Label',
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.yaml';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processYamlFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should parse YAML and return stringified JSON', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+key1: value1
+key2: value2
+    `;
+    const expectedJson = JSON.stringify({ key1: 'value1', key2: 'value2' });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should handle YAML with nested structures', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+parent:
+  child1: value1
+  child2: value2
+array:
+  - item1
+  - item2
+    `;
+    const expectedJson = JSON.stringify({
+      parent: { child1: 'value1', child2: 'value2' },
+      array: ['item1', 'item2'],
+    });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+  });
+
+  it('should handle YAML with whitespace in values', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+key: "value with    spaces"
+template: "{{ variable }}   "
+    `;
+    const expectedJson = JSON.stringify({
+      key: 'value with    spaces',
+      template: '{{ variable }}   ',
+    });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+  });
+
+  it('should handle invalid YAML and return raw file contents', () => {
+    const filePath = 'issue-2368.yaml';
+    const fileContent = dedent`
+    {% import "system_prompt.yaml" as system_prompt %}
+    {% import "user_prompt.yaml" as user_prompt %}
+    {{ system_prompt.system_prompt() }}
+    {{ user_prompt.user_prompt(example) }}`;
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    expect(processYamlFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 80)}`,
+        config: undefined,
+      },
+    ]);
+    expect(logger.debug).toHaveBeenCalledWith(
+      expect.stringMatching(/Error parsing YAML file issue-2368\.yaml:/),
+    );
+  });
+
+  describe('recursive file:// resolution', () => {
+    it('should recursively resolve file:// references in YAML content', () => {
+      const filePath = 'conversation.yaml';
+      const fileContent = dedent`
+        - role: system
+          content: file://system.md
+        - role: user
+          content: file://user.md
+      `;
+
+      const parsedYaml = [
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ];
+
+      const resolvedContent = [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'What is 2 + 2?' },
+      ];
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle nested file:// references in complex structures', () => {
+      const filePath = 'config.yaml';
+      const fileContent = dedent`
+        provider:
+          id: openai:gpt-4
+          config:
+            temperature: file://temperature.json
+            tools: file://tools.yaml
+        messages:
+          - role: system
+            content: file://system.txt
+      `;
+
+      const parsedYaml = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.yaml',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      };
+
+      const resolvedContent = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 0.7,
+            tools: [{ name: 'search', description: 'Search the web' }],
+          },
+        },
+        messages: [{ role: 'system', content: 'You are a helpful assistant.' }],
+      };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle when maybeLoadConfigFromExternalFile returns unchanged content', () => {
+      const filePath = 'no-refs.yaml';
+      const fileContent = dedent`
+        key1: value1
+        key2: value2
+      `;
+
+      const parsedYaml = { key1: 'value1', key2: 'value2' };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(parsedYaml);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(parsedYaml));
+    });
+
+    it('should preserve empty string values when parsing YAML', () => {
+      const filePath = 'empty-value.yaml';
+      const fileContent = dedent`
+        prompts:
+          - key: ""
+      `;
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      const parsed = JSON.parse(result[0].raw as string);
+      expect(parsed).toEqual({ prompts: [{ key: '' }] });
+    });
+
+    it('should preserve null values when parsing YAML', () => {
+      const filePath = 'null-value.yaml';
+      const fileContent = dedent`
+        prompts:
+          - key: null
+        options:
+          temperature: ~
+      `;
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      const parsed = JSON.parse(result[0].raw as string);
+      expect(parsed).toEqual({
+        prompts: [{ key: null }],
+        options: { temperature: null },
+      });
+    });
+
+    it('should handle empty YAML document', () => {
+      const filePath = 'empty.yaml';
+      const fileContent = '';
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      // Empty YAML parses to undefined; maybeLoadConfigFromExternalFile passes it
+      // through; JSON.stringify(undefined) is the literal string "undefined".
+      expect(result[0].raw).toBe(JSON.stringify(undefined));
+    });
+  });
+});

test/prompts/utils.test.ts

@@ -0,0 +1,234 @@
+import { describe, expect, it, vi } from 'vitest';
+import { hashPrompt, maybeFilePath, normalizeInput } from '../../src/prompts/utils';
+import { sha256 } from '../../src/util/createHash';
+
+vi.mock('fs', async () => {
+  const actual = await vi.importActual<typeof import('fs')>('fs');
+  return {
+    ...actual,
+    statSync: vi.fn(actual.statSync),
+  };
+});
+
+vi.mock('glob', () => ({
+  globSync: vi.fn(),
+  hasMagic: vi.fn((pattern: string | string[]) => {
+    const p = Array.isArray(pattern) ? pattern.join('') : pattern;
+    return p.includes('*') || p.includes('?') || p.includes('[') || p.includes('{');
+  }),
+}));
+
+describe('maybeFilePath', () => {
+  it('should return true for valid file paths', () => {
+    expect(maybeFilePath('C:\\path\\to\\file.txt')).toBe(true);
+    expect(maybeFilePath('file.*')).toBe(true);
+    expect(maybeFilePath('filename.ext')).toBe(true);
+    expect(maybeFilePath('path/to/file.txt')).toBe(true);
+  });
+
+  it('should return false for strings with new lines', () => {
+    expect(maybeFilePath('path/to\nfile.txt')).toBe(false);
+    expect(maybeFilePath('file\nname.ext')).toBe(false);
+  });
+
+  it('should return false for strings with "portkey://"', () => {
+    expect(maybeFilePath('portkey://path/to/file')).toBe(false);
+  });
+
+  it('should return false for strings with "langfuse://"', () => {
+    expect(maybeFilePath('langfuse://path/to/file')).toBe(false);
+  });
+
+  it('should return false for strings with "helicone://"', () => {
+    expect(maybeFilePath('helicone://path/to/file')).toBe(false);
+  });
+
+  it('should return false for strings without file path indicators', () => {
+    expect(maybeFilePath('anotherstring')).toBe(false);
+    expect(maybeFilePath('justastring')).toBe(false);
+    expect(maybeFilePath('stringwith.dotbutnotfile')).toBe(false);
+  });
+
+  it('should return true for strings with file:// prefix', () => {
+    expect(maybeFilePath('file://path/to/file.txt')).toBe(true);
+  });
+
+  it('should return true for strings with wildcard character', () => {
+    expect(maybeFilePath('*.txt')).toBe(true);
+    expect(maybeFilePath('path/to/*.txt')).toBe(true);
+  });
+
+  it('should return true for strings with file extension at the third or fourth last position', () => {
+    expect(maybeFilePath('file.ext')).toBe(true);
+    expect(maybeFilePath('file.name.ext')).toBe(true);
+    expect(maybeFilePath('filename.e')).toBe(false);
+    expect(maybeFilePath('filename.ex')).toBe(true);
+  });
+
+  it('should work for files that end with specific allowed extensions', () => {
+    expect(maybeFilePath('filename.cjs')).toBe(true);
+    expect(maybeFilePath('filename.js')).toBe(true);
+    expect(maybeFilePath('filename.js:functionName')).toBe(true);
+    expect(maybeFilePath('filename.j2')).toBe(true);
+    expect(maybeFilePath('filename.json')).toBe(true);
+    expect(maybeFilePath('filename.jsonl')).toBe(true);
+    expect(maybeFilePath('filename.mjs')).toBe(true);
+    expect(maybeFilePath('filename.py')).toBe(true);
+    expect(maybeFilePath('filename.py:functionName')).toBe(true);
+    expect(maybeFilePath('filename.txt')).toBe(true);
+  });
+
+  it('should recognize function references when the path is not the first colon-delimited token', () => {
+    expect(maybeFilePath('label:path/to/filename.py:functionName')).toBe(true);
+    expect(maybeFilePath('label:path/to/filename.js:functionName')).toBe(true);
+  });
+
+  // Additional tests
+  it('should return false for empty strings', () => {
+    expect(maybeFilePath('')).toBe(false);
+  });
+
+  it('should return false for whitespace strings', () => {
+    expect(maybeFilePath('   ')).toBe(false);
+    expect(maybeFilePath('\t')).toBe(false);
+  });
+
+  it('should return false for non-string inputs', () => {
+    expect(() => maybeFilePath(123 as never)).toThrow('Invalid input: 123');
+    expect(() => maybeFilePath({} as never)).toThrow('Invalid input: {}');
+    expect(() => maybeFilePath([] as never)).toThrow('Invalid input: []');
+  });
+
+  it('should return false for strings with invalid and valid indicators mixed', () => {
+    expect(maybeFilePath('file://path/to\nfile.txt')).toBe(false);
+    expect(maybeFilePath('path/to/file.txtportkey://')).toBe(false);
+  });
+
+  it('should return true for very long valid file paths', () => {
+    const longPath = `${'a/'.repeat(100)}file.txt`;
+    expect(maybeFilePath(longPath)).toBe(true);
+  });
+
+  it('should return false for very long invalid file paths', () => {
+    const longInvalidPath = `${'a/'.repeat(100)}file\n.txt`;
+    expect(maybeFilePath(longInvalidPath)).toBe(false);
+  });
+
+  it('should return false for strings ending with a dot', () => {
+    expect(maybeFilePath('Write a tweet about {{topic}}.')).toBe(false);
+  });
+
+  it('should recognize Jinja2 template files', () => {
+    expect(maybeFilePath('template.j2')).toBe(true);
+    expect(maybeFilePath('path/to/template.j2')).toBe(true);
+    expect(maybeFilePath('file://path/to/template.j2')).toBe(true);
+    expect(maybeFilePath('*.j2')).toBe(true);
+    expect(maybeFilePath('path/to/*.j2')).toBe(true);
+    expect(maybeFilePath('template.j2:functionName')).toBe(true);
+  });
+
+  it('should return false for prompt strings resembling Jinja2 syntax', () => {
+    expect(maybeFilePath('Hello {{ name }}! How are you?')).toBe(false);
+    expect(maybeFilePath('{% if condition %}Content{% endif %}')).toBe(false);
+    expect(maybeFilePath('{{ variable | filter }}')).toBe(false);
+    expect(maybeFilePath('Text with {{ variable.attribute }} in it.')).toBe(false);
+  });
+});
+
+describe('normalizeInput', () => {
+  it('rejects invalid input types', () => {
+    expect(() => normalizeInput(null as any)).toThrow('Invalid input prompt: null');
+    expect(() => normalizeInput(undefined as any)).toThrow('Invalid input prompt: undefined');
+    expect(() => normalizeInput(1 as any)).toThrow('Invalid input prompt: 1');
+    expect(() => normalizeInput(true as any)).toThrow('Invalid input prompt: true');
+    expect(() => normalizeInput(false as any)).toThrow('Invalid input prompt: false');
+  });
+
+  it('rejects empty inputs', () => {
+    expect(() => normalizeInput([])).toThrow('Invalid input prompt: []');
+    expect(() => normalizeInput({} as any)).toThrow('Invalid input prompt: {}');
+    expect(() => normalizeInput('')).toThrow('Invalid input prompt: ""');
+  });
+
+  it('returns array with single string when input is a non-empty string', () => {
+    expect(normalizeInput('valid string')).toEqual([{ raw: 'valid string' }]);
+  });
+
+  it('returns input array when input is a non-empty array', () => {
+    const inputArray = ['prompt1', { raw: 'prompt2' }];
+    expect(normalizeInput(inputArray)).toEqual([{ raw: 'prompt1' }, { raw: 'prompt2' }]);
+  });
+
+  // NOTE: Legacy mode. This is deprecated and will be removed in a future version.
+  it('returns array of prompts when input is an object', () => {
+    const inputObject = {
+      'prompts1.txt': 'label A',
+      'prompts2.txt': 'label B',
+    };
+    expect(normalizeInput(inputObject)).toEqual([
+      {
+        label: 'label A',
+        raw: 'prompts1.txt',
+      },
+      {
+        label: 'label B',
+        raw: 'prompts2.txt',
+      },
+    ]);
+  });
+});
+
+describe('hashPrompt', () => {
+  describe('priority order', () => {
+    it('uses label when provided (highest priority)', () => {
+      expect(
+        hashPrompt({
+          id: 'prompt-id',
+          label: 'Prompt Label',
+          raw: 'Prompt Raw',
+        }),
+      ).toBe(sha256('Prompt Label'));
+    });
+
+    it('falls back to id when label is empty string', () => {
+      expect(
+        hashPrompt({
+          id: 'prompt-id',
+          label: '',
+          raw: 'Prompt Raw',
+        }),
+      ).toBe(sha256('prompt-id'));
+    });
+
+    it('falls back to raw when both label and id are empty', () => {
+      expect(
+        hashPrompt({
+          id: '',
+          label: '',
+          raw: 'Prompt Raw Content',
+        }),
+      ).toBe(sha256('Prompt Raw Content'));
+    });
+
+    it('falls back to raw when label and id are not provided', () => {
+      expect(
+        hashPrompt({
+          label: '',
+          raw: 'Only raw content',
+        }),
+      ).toBe(sha256('Only raw content'));
+    });
+  });
+
+  describe('consistency with generateIdFromPrompt', () => {
+    it('produces same result as generateIdFromPrompt', () => {
+      const prompt = {
+        id: 'test-id',
+        label: 'Test Label',
+        raw: 'Test Raw',
+      };
+      // Both functions should produce identical results
+      expect(hashPrompt(prompt)).toBe(sha256('Test Label'));
+    });
+  });
+});

test/prompts/wildcard.test.ts

@@ -0,0 +1,151 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processPrompts } from '../../src/prompts/index';
+
+// Mock the python execution
+vi.mock('../../src/python/pythonUtils', () => ({
+  runPython: vi.fn((filePath: string, _functionName: string, _args: string[]) => {
+    // Return different responses based on file path
+    if (filePath.includes('marketing')) {
+      return Promise.resolve(
+        JSON.stringify({
+          output: `Marketing prompt from ${filePath}`,
+        }),
+      );
+    } else if (filePath.includes('technical')) {
+      return Promise.resolve(
+        JSON.stringify({
+          output: `Technical prompt from ${filePath}`,
+        }),
+      );
+    }
+    return Promise.resolve(
+      JSON.stringify({
+        output: 'Default prompt response',
+      }),
+    );
+  }),
+}));
+
+// Mock fs operations
+vi.mock('fs', () => ({
+  existsSync: vi.fn(() => true),
+  readFileSync: vi.fn((path: string) => {
+    if (path.includes('.py')) {
+      return 'def get_prompt(context):\n    return "Test prompt"';
+    }
+    if (path.includes('.js')) {
+      return 'module.exports = function() { return "JS prompt"; };';
+    }
+    return '';
+  }),
+  statSync: vi.fn(() => ({
+    size: 1000,
+    isDirectory: () => false,
+  })),
+  writeFileSync: vi.fn(),
+}));
+
+// Mock glob results
+vi.mock('glob', () => ({
+  globSync: vi.fn((pattern: string) => {
+    if (pattern.includes('test/prompts/**/*.py')) {
+      return [
+        'test/prompts/marketing/email.py',
+        'test/prompts/marketing/social.py',
+        'test/prompts/technical/review.py',
+      ];
+    }
+    if (pattern.includes('test/prompts/**/*.js')) {
+      return ['test/prompts/ui/button.js', 'test/prompts/ui/form.js'];
+    }
+    return [];
+  }),
+  hasMagic: vi.fn((pattern: string | string[]) => {
+    const p = Array.isArray(pattern) ? pattern.join('') : pattern;
+    return p.includes('*') || p.includes('?') || p.includes('[') || p.includes('{');
+  }),
+}));
+
+// Mock importModule for JavaScript files
+vi.mock('../../src/esm', () => ({
+  importModule: vi.fn((filePath: string) => {
+    return Promise.resolve(function (_context: any) {
+      return `JS prompt from ${filePath}`;
+    });
+  }),
+}));
+
+describe('Wildcard prompt support', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('Python wildcards', () => {
+    it('should expand wildcard patterns and preserve function names', async () => {
+      const prompts = await processPrompts(['file://test/prompts/**/*.py:get_prompt']);
+
+      // Should create one prompt for each matched file
+      expect(prompts.length).toBe(3);
+
+      // Each prompt should have the function name preserved
+      prompts.forEach((prompt) => {
+        expect(prompt.label).toMatch(/\.py:get_prompt$/);
+      });
+    });
+
+    it('should work without function names', async () => {
+      const prompts = await processPrompts(['file://test/prompts/**/*.py']);
+
+      expect(prompts.length).toBe(3);
+
+      // For Python files without function names, labels include file content
+      prompts.forEach((prompt) => {
+        expect(prompt.label).toContain('.py: ');
+      });
+    });
+  });
+
+  describe('JavaScript wildcards', () => {
+    it('should expand wildcard patterns for JavaScript files', async () => {
+      const prompts = await processPrompts(['file://test/prompts/**/*.js']);
+
+      expect(prompts.length).toBe(2);
+
+      prompts.forEach((prompt) => {
+        expect(prompt.label).toMatch(/\.js$/);
+      });
+    });
+
+    it('should preserve function names for JavaScript', async () => {
+      const prompts = await processPrompts(['file://test/prompts/**/*.js:myFunction']);
+
+      expect(prompts.length).toBe(2);
+
+      prompts.forEach((prompt) => {
+        expect(prompt.label).toMatch(/\.js:myFunction$/);
+      });
+    });
+  });
+
+  describe('Mixed patterns', () => {
+    it('should handle both wildcards and explicit paths', async () => {
+      // Mock for the explicit path
+      vi.mocked(fs.existsSync).mockImplementation((path) => {
+        if (String(path).includes('explicit.py')) {
+          return true;
+        }
+        return true;
+      });
+
+      const prompts = await processPrompts([
+        'file://test/prompts/**/*.py:get_prompt',
+        'file://test/explicit.py:another_function',
+      ]);
+
+      // 3 from wildcard + 1 explicit
+      expect(prompts.length).toBe(4);
+    });
+  });
+});