chore: import upstream snapshot with attribution

test/prompts/processors/csv.test.ts

@@ -0,0 +1,242 @@
+import fs from 'fs';
+
+import dedent from 'ts-dedent';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processCsvPrompts } from '../../../src/prompts/processors/csv';
+import { mockProcessEnv } from '../../util/utils';
+
+import type { Prompt } from '../../../src/types/index';
+
+vi.mock('fs');
+
+describe('processCsvPrompts', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should process a single column CSV with header', async () => {
+    const csvContent = dedent`
+      prompt
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Prompt 1 - Tell me about {{topic}}',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Prompt 2 - Explain {{topic}} in simple terms',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Prompt 3 - Write a poem about {{topic}}',
+      },
+    ]);
+  });
+
+  it('should process a single column text file without header', async () => {
+    const csvContent = dedent`
+      Tell me about {{topic}}
+      Explain {{topic}} in simple terms
+      Write a poem about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Prompt 1 - Tell me about {{topic}}',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Prompt 2 - Explain {{topic}} in simple terms',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Prompt 3 - Write a poem about {{topic}}',
+      },
+    ]);
+  });
+
+  it('should process a two column CSV with prompt and label', async () => {
+    const csvContent = dedent`
+      prompt,label
+      Tell me about {{topic}},Basic Query
+      Explain {{topic}} in simple terms,Simple Explanation
+      Write a poem about {{topic}},Poetry Generator
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Basic Query',
+      },
+      {
+        raw: 'Explain {{topic}} in simple terms',
+        label: 'Simple Explanation',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Poetry Generator',
+      },
+    ]);
+  });
+
+  it('should generate labels from prompt content when not provided', async () => {
+    const csvContent = dedent`
+      prompt
+      This is a very long prompt that should be truncated for the label
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(1);
+    expect(result).toEqual([
+      {
+        raw: 'This is a very long prompt that should be truncated for the label',
+        label: 'Prompt 1 - This is a very long prompt that should be truncated for the label',
+      },
+    ]);
+  });
+
+  it('should use base prompt properties if provided', async () => {
+    const csvContent = dedent`
+      prompt
+      Tell me about {{topic}}
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const basePrompt: Partial<Prompt> = {
+      label: 'Base Label',
+      id: 'base-id',
+    };
+
+    const result = await processCsvPrompts('prompts.csv', basePrompt);
+
+    expect(result).toHaveLength(1);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Base Label',
+        id: 'base-id',
+      },
+    ]);
+  });
+
+  it('should skip rows with missing prompt values', async () => {
+    const csvContent = dedent`
+      prompt,label
+      Tell me about {{topic}},Basic Query
+
+      Write a poem about {{topic}},Poetry Generator
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(2);
+    expect(result).toEqual([
+      {
+        raw: 'Tell me about {{topic}}',
+        label: 'Basic Query',
+      },
+      {
+        raw: 'Write a poem about {{topic}}',
+        label: 'Poetry Generator',
+      },
+    ]);
+  });
+
+  it('should handle custom delimiters', async () => {
+    const csvContent = dedent`
+      prompt;label
+      Tell me about {{topic}};Basic Query
+      Explain {{topic}} in simple terms;Simple Explanation
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+    const restoreEnv = mockProcessEnv({ PROMPTFOO_CSV_DELIMITER: ';' });
+    try {
+      const result = await processCsvPrompts('prompts.csv', {});
+
+      expect(result).toHaveLength(2);
+      expect(result).toEqual([
+        {
+          raw: 'Tell me about {{topic}}',
+          label: 'Basic Query',
+        },
+        {
+          raw: 'Explain {{topic}} in simple terms',
+          label: 'Simple Explanation',
+        },
+      ]);
+    } finally {
+      restoreEnv();
+    }
+  });
+
+  it('should handle empty files', async () => {
+    vi.mocked(fs.readFileSync).mockReturnValue('');
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle malformed CSV by falling back to line-by-line processing', async () => {
+    const csvContent = dedent`
+      "prompt","label"
+      "Tell me about {{topic}},"Basic Query"
+      "Malformed line with unbalanced quotes
+      "Another line
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(4);
+    expect(result[0].raw).toBe('"prompt","label"');
+    expect(result[1].raw).toBe('"Tell me about {{topic}},"Basic Query"');
+    expect(result[2].raw).toBe('"Malformed line with unbalanced quotes');
+    expect(result[3].raw).toBe('"Another line');
+  });
+
+  it('should handle malformed CSV with a header row correctly', async () => {
+    const csvContent = dedent`
+      prompt
+      "Malformed CSV with a quote problem
+      Another prompt line
+    `;
+
+    vi.mocked(fs.readFileSync).mockReturnValue(csvContent);
+
+    const result = await processCsvPrompts('prompts.csv', {});
+
+    expect(result).toHaveLength(2);
+    expect(result[0].raw).toBe('"Malformed CSV with a quote problem');
+    expect(result[1].raw).toBe('Another prompt line');
+  });
+});

test/prompts/processors/executable.test.ts

@@ -0,0 +1,199 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { processExecutableFile } from '../../../src/prompts/processors/executable';
+
+import type { ApiProvider } from '../../../src/types/index';
+
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+vi.mock('../../../src/cache', () => ({
+  getCache: vi.fn(() => ({
+    get: vi.fn(),
+    set: vi.fn(),
+  })),
+  isCacheEnabled: vi.fn(() => false),
+}));
+
+describe('processExecutableFile', () => {
+  const mockProvider = {
+    id: vi.fn(() => 'test-provider'),
+    label: 'Test Provider',
+    callApi: vi.fn(),
+  } as ApiProvider;
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  const describeUnix = process.platform === 'win32' ? describe.skip : describe;
+
+  // Cross-platform tests
+  it('should process a script with exec: prefix', async () => {
+    const scriptPath =
+      process.platform === 'win32' ? 'cmd.exe /c echo "test"' : '/usr/bin/echo "test"';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    expect(prompts[0].raw).toBe(scriptPath);
+    expect(typeof prompts[0].function).toBe('function');
+  });
+
+  it('should use custom label when provided', async () => {
+    const scriptPath = process.platform === 'win32' ? 'cmd.exe' : '/bin/echo';
+    const prompts = await processExecutableFile(scriptPath, { label: 'Custom Label' });
+
+    expect(prompts[0].label).toBe('Custom Label');
+  });
+
+  it('should handle binary executables', async () => {
+    const scriptPath =
+      process.platform === 'win32' ? 'C:\\Windows\\System32\\cmd.exe' : '/usr/bin/ls';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    // Binary files should not be read as raw content
+    expect(prompts[0].raw).toBe(scriptPath);
+  });
+
+  it('should handle non-existent files gracefully', async () => {
+    const scriptPath = '/non/existent/script.sh';
+    const prompts = await processExecutableFile(scriptPath, {});
+
+    expect(prompts).toHaveLength(1);
+    expect(prompts[0].label).toBe(scriptPath);
+    expect(prompts[0].raw).toBe(scriptPath);
+  });
+
+  // Unix-specific tests
+  describeUnix('Unix shell script tests', () => {
+    let sharedPrompts: Awaited<ReturnType<typeof processExecutableFile>>;
+    let tempDir: string;
+    let scriptPath: string;
+
+    beforeAll(async () => {
+      tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-executable-test-'));
+      scriptPath = path.join(tempDir, 'shared-test-script.sh');
+      fs.writeFileSync(
+        scriptPath,
+        `#!/bin/sh
+context="$1"
+case "$context" in
+  *'"mode":"args"'*)
+    printf '%s\\n' "Context received: $context"
+    ;;
+  *'"mode":"config"'*)
+    printf '%s\\n' "Test output"
+    ;;
+  *'"mode":"stderr"'*)
+    printf '%s\\n' "Error message" >&2
+    printf '%s\\n' "Normal output"
+    ;;
+  *'"mode":"error"'*)
+    printf '%s\\n' "Error only" >&2
+    exit 1
+    ;;
+  *'"mode":"relative"'*)
+    printf '%s\\n' "Relative path works"
+    ;;
+  *)
+    printf '%s\\n' "Hello from shell script"
+    ;;
+esac
+`,
+      );
+      fs.chmodSync(scriptPath, 0o755);
+      sharedPrompts = await processExecutableFile(scriptPath, {});
+    });
+
+    afterAll(() => {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    });
+
+    it('should process a simple shell script', async () => {
+      expect(sharedPrompts).toHaveLength(1);
+      expect(sharedPrompts[0].label).toBe(scriptPath);
+      expect(sharedPrompts[0].raw).toContain('#!/bin/sh');
+      expect(typeof sharedPrompts[0].function).toBe('function');
+
+      const result = await sharedPrompts[0].function!({
+        vars: { test: 'value' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Hello from shell script');
+    });
+
+    it('should handle scripts with arguments', async () => {
+      const result = await sharedPrompts[0].function!({
+        vars: { mode: 'args', name: 'test' },
+        provider: mockProvider,
+      });
+
+      // The script should receive the context as JSON
+      expect(result).toContain('Context received:');
+      expect(result).toContain('"vars"');
+      expect(result).toContain('"name":"test"');
+    });
+
+    it('should pass config to the function', async () => {
+      const config = { temperature: 0.5 };
+      const prompts = await processExecutableFile(scriptPath, { config });
+
+      expect(prompts[0].config).toEqual(config);
+
+      const result = await prompts[0].function!({
+        vars: { mode: 'config' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Test output');
+    });
+
+    it('should handle scripts that output to stderr', async () => {
+      const result = await sharedPrompts[0].function!({
+        vars: { mode: 'stderr' },
+        provider: mockProvider,
+      });
+
+      // Should return stdout even if there's stderr
+      expect(result).toBe('Normal output');
+    });
+
+    it('should reject when script fails with no stdout', async () => {
+      await expect(
+        sharedPrompts[0].function!({
+          vars: { mode: 'error' },
+          provider: mockProvider,
+        }),
+      ).rejects.toThrow();
+    });
+
+    it('should resolve relative paths from prompt.config.basePath', async () => {
+      const relativeScriptPath = `.${path.sep}${path.basename(scriptPath)}`;
+      const prompts = await processExecutableFile(relativeScriptPath, {
+        config: { basePath: tempDir },
+      });
+
+      expect(prompts).toHaveLength(1);
+      expect(prompts[0].label).toBe(relativeScriptPath);
+      expect(typeof prompts[0].function).toBe('function');
+
+      const result = await prompts[0].function!({
+        vars: { mode: 'relative' },
+        provider: mockProvider,
+      });
+
+      expect(result).toBe('Relative path works');
+    });
+  });
+});

test/prompts/processors/javascript.test.ts

@@ -0,0 +1,157 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { importModule } from '../../../src/esm';
+import { processJsFile, transformContext } from '../../../src/prompts/processors/javascript';
+import invariant from '../../../src/util/invariant';
+
+vi.mock('../../../src/esm', () => ({
+  importModule: vi.fn(),
+}));
+
+describe('transformContext', () => {
+  it('should transform context with provider and config', () => {
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel', callApi: vi.fn() },
+      config: { configKey: 'configValue' },
+    };
+    const result = transformContext(context);
+    expect(result).toEqual({
+      vars: { key: 'value' },
+      provider: { id: 'providerId', label: 'providerLabel' },
+      config: { configKey: 'configValue' },
+    });
+  });
+
+  it('should transform context with provider and empty config', () => {
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel', callApi: vi.fn() },
+    };
+    const result = transformContext(context);
+    expect(result).toEqual({
+      vars: { key: 'value' },
+      provider: { id: 'providerId', label: 'providerLabel' },
+      config: {},
+    });
+  });
+
+  it('should throw an error if provider is missing', () => {
+    const context = { vars: { key: 'value' } };
+    expect(() => transformContext(context)).toThrow('Provider is required');
+  });
+});
+
+describe('processJsFile', () => {
+  const mockImportModule = vi.mocked(importModule);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid JavaScript file without a function name or a label', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, {}, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with a label without a function name', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, { label: 'myLabel' }, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'myLabel',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with a function name without a label', async () => {
+    const filePath = 'file.js';
+    const functionName = 'myFunction';
+    const mockFunction = () => 'dummy prompt';
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, {}, functionName)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js:myFunction',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, functionName);
+  });
+
+  it('should process a valid JavaScript file with a label and a function name', async () => {
+    const filePath = 'file.js';
+    const functionName = 'myFunction';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    await expect(processJsFile(filePath, { label: 'myLabel' }, functionName)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'myLabel',
+        function: expect.any(Function),
+        config: {},
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, functionName);
+  });
+
+  it('should throw an error if the file cannot be imported', async () => {
+    const filePath = 'nonexistent.js';
+    mockImportModule.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    await expect(processJsFile(filePath, {}, undefined)).rejects.toThrow('File not found');
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should process a valid JavaScript file with config', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    const config = { key: 'value' };
+    await expect(processJsFile(filePath, { config }, undefined)).resolves.toEqual([
+      {
+        raw: String(mockFunction),
+        label: 'file.js',
+        function: expect.any(Function),
+        config: { key: 'value' },
+      },
+    ]);
+    expect(mockImportModule).toHaveBeenCalledWith(filePath, undefined);
+  });
+
+  it('should pass config to the prompt function', async () => {
+    const filePath = 'file.js';
+    const mockFunction = vi.fn(() => 'dummy prompt');
+    mockImportModule.mockResolvedValue(mockFunction);
+    const config = { key: 'value' };
+    const result = await processJsFile(filePath, { config }, undefined);
+    const promptFunction = result[0].function;
+    invariant(promptFunction, 'Prompt function is required');
+    await promptFunction({
+      vars: {},
+      provider: { id: () => 'test', label: 'Test', callApi: vi.fn() },
+    });
+    expect(mockFunction).toHaveBeenCalledWith(
+      expect.objectContaining({
+        config: { key: 'value' },
+      }),
+    );
+  });
+});

test/prompts/processors/jinja.test.ts

@@ -0,0 +1,102 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJinjaFile } from '../../../src/prompts/processors/jinja';
+
+vi.mock('fs');
+
+describe('processJinjaFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a Jinja2 file without a label', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, {});
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a Jinja2 file with a label', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, { label: 'Custom Label' });
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: 'Custom Label',
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should include config when provided', () => {
+    const filePath = 'template.j2';
+    const fileContent =
+      'You are a helpful assistant for Promptfoo.\nPlease answer the following question about {{ topic }}: {{ question }}';
+    const config = { temperature: 0.7, max_tokens: 150 };
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, { config });
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.j2';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processJinjaFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should handle variable interpolation syntax properly', () => {
+    const filePath = 'complex.j2';
+    const fileContent = `
+    {% if condition %}
+      Handle {{ variable1 }} with condition
+    {% else %}
+      Handle {{ variable2 }} without condition
+    {% endif %}
+    `;
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processJinjaFile(filePath, {});
+
+    expect(result).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 50)}...`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+});

test/prompts/processors/json.test.ts

@@ -0,0 +1,155 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJsonFile } from '../../../src/prompts/processors/json';
+import * as fileModule from '../../../src/util/file';
+
+vi.mock('fs');
+vi.mock('../../../src/util/file');
+
+describe('processJsonFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+  const mockMaybeLoadConfigFromExternalFile = vi.mocked(fileModule.maybeLoadConfigFromExternalFile);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // By default, maybeLoadConfigFromExternalFile returns its input unchanged
+    mockMaybeLoadConfigFromExternalFile.mockImplementation((input) => input);
+  });
+
+  it('should process a valid JSON file without a label', () => {
+    const filePath = 'file.json';
+    const fileContent = JSON.stringify({ key: 'value' });
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent}`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid JSON file with a label', () => {
+    const filePath = 'file.json';
+    const fileContent = JSON.stringify({ key: 'value' });
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: fileContent,
+        label: `Label`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.json';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processJsonFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  describe('recursive file:// resolution', () => {
+    it('should recursively resolve file:// references in JSON content', () => {
+      const filePath = 'conversation.json';
+      const fileContent = JSON.stringify([
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ]);
+
+      const parsedJson = [
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ];
+
+      const resolvedContent = [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'What is 2 + 2?' },
+      ];
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle nested file:// references in complex JSON structures', () => {
+      const filePath = 'config.json';
+      const fileContent = JSON.stringify({
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.json',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      });
+
+      const parsedJson = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.json',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      };
+
+      const resolvedContent = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 0.7,
+            tools: [{ name: 'search', description: 'Search the web' }],
+          },
+        },
+        messages: [{ role: 'system', content: 'You are a helpful assistant.' }],
+      };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle invalid JSON and return original content', () => {
+      const filePath = 'invalid.json';
+      const fileContent = 'This is not valid JSON {]';
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processJsonFile(filePath, {});
+
+      // When JSON parsing fails, it should return the original content
+      expect(mockMaybeLoadConfigFromExternalFile).not.toHaveBeenCalled();
+      expect(result[0].raw).toBe(fileContent);
+    });
+
+    it('should handle when maybeLoadConfigFromExternalFile returns unchanged content', () => {
+      const filePath = 'no-refs.json';
+      const fileContent = JSON.stringify({ key1: 'value1', key2: 'value2' });
+
+      const parsedJson = { key1: 'value1', key2: 'value2' };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(parsedJson);
+
+      const result = processJsonFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedJson);
+      expect(result[0].raw).toBe(JSON.stringify(parsedJson));
+    });
+  });
+});

test/prompts/processors/jsonl.test.ts

@@ -0,0 +1,83 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processJsonlFile } from '../../../src/prompts/processors/jsonl';
+
+vi.mock('fs');
+
+describe('processJsonlFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid JSONL file without a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}]\n[{"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, {})).toEqual([
+      {
+        raw: '[{"key1": "value1"}]',
+        label: 'file.jsonl: [{"key1": "value1"}]',
+      },
+      {
+        raw: '[{"key2": "value2"}]',
+        label: 'file.jsonl: [{"key2": "value2"}]',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with a single record without a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}, {"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, {})).toEqual([
+      {
+        raw: '[{"key1": "value1"}, {"key2": "value2"}]',
+        label: `file.jsonl`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with a single record and a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}, {"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: '[{"key1": "value1"}, {"key2": "value2"}]',
+        label: `Label`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a valid JSONL file with multiple records and a label', () => {
+    const filePath = 'file.jsonl';
+    const fileContent = '[{"key1": "value1"}]\n[{"key2": "value2"}]';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processJsonlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: '[{"key1": "value1"}]',
+        label: `Label: [{"key1": "value1"}]`,
+      },
+      {
+        raw: '[{"key2": "value2"}]',
+        label: `Label: [{"key2": "value2"}]`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.jsonl';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+    expect(() => processJsonlFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+});

test/prompts/processors/markdown.test.ts

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { processMarkdownFile } from '../../../src/prompts/processors/markdown';
+
+vi.mock('fs');
+
+describe('processMarkdownFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a valid Markdown file without a label', () => {
+    const filePath = 'file.md';
+    const fileContent = '# Heading\n\nThis is some markdown content.';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: # Heading\n\nThis is some markdown content....`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid Markdown file with a label', () => {
+    const filePath = 'file.md';
+    const fileContent = '# Heading\n\nThis is some markdown content.';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, { label: 'Custom Label' })).toEqual([
+      {
+        raw: fileContent,
+        label: 'Custom Label',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should truncate the label for long Markdown files', () => {
+    const filePath = 'file.md';
+    const fileContent = '# ' + 'A'.repeat(100);
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processMarkdownFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: # ${'A'.repeat(48)}...`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.md';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processMarkdownFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+});

test/prompts/processors/python.test.ts

@@ -0,0 +1,99 @@
+import * as fs from 'fs';
+
+import dedent from 'dedent';
+import { describe, expect, it, vi } from 'vitest';
+import {
+  processPythonFile,
+  pythonPromptFunction,
+  pythonPromptFunctionLegacy,
+} from '../../../src/prompts/processors/python';
+
+vi.mock('fs');
+vi.mock('../../../src/prompts/processors/python', async () => {
+  const actual = await vi.importActual('../../../src/prompts/processors/python');
+  return {
+    ...actual,
+    pythonPromptFunction: vi.fn(),
+    pythonPromptFunctionLegacy: vi.fn(),
+  };
+});
+
+describe('processPythonFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  it('should process a valid Python file without a function name or label', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, {}, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `${filePath}: ${fileContent}`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a function name without a label', () => {
+    const filePath = 'file.py';
+    const fileContent = dedent`
+    def testFunction(context):
+      print("Hello, world!")`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunction).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, {}, 'testFunction')).toEqual([
+      {
+        function: expect.any(Function),
+        raw: fileContent,
+        label: `file.py:testFunction`,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a label without a function name', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, { label: 'Label' }, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `Label`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with a label and function name', () => {
+    const filePath = 'file.py';
+    const fileContent = dedent`
+    def testFunction(context):
+      print("Hello, world!")`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunction).mockResolvedValueOnce('mocked result');
+    expect(processPythonFile(filePath, { label: 'Label' }, 'testFunction')).toEqual([
+      {
+        function: expect.any(Function),
+        label: `Label`,
+        raw: fileContent,
+      },
+    ]);
+  });
+
+  it('should process a valid Python file with config', () => {
+    const filePath = 'file.py';
+    const fileContent = 'print("Hello, world!")';
+    mockReadFileSync.mockReturnValue(fileContent);
+    vi.mocked(pythonPromptFunctionLegacy).mockResolvedValueOnce('mocked result');
+    const config = { key: 'value' };
+    expect(processPythonFile(filePath, { config }, undefined)).toEqual([
+      {
+        function: expect.any(Function),
+        label: `${filePath}: ${fileContent}`,
+        raw: fileContent,
+        config: { key: 'value' },
+      },
+    ]);
+  });
+});

test/prompts/processors/python.utils.test.ts

@@ -0,0 +1,91 @@
+import { PythonShell } from 'python-shell';
+import { describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import {
+  pythonPromptFunction,
+  pythonPromptFunctionLegacy,
+} from '../../../src/prompts/processors/python';
+import { runPython } from '../../../src/python/pythonUtils';
+
+import type { ApiProvider } from '../../../src/types/index';
+
+vi.mock('fs');
+vi.mock('python-shell');
+vi.mock('../../../src/python/pythonUtils');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+  },
+}));
+
+describe('pythonPromptFunction', () => {
+  interface PythonContext {
+    vars: Record<string, string | object>;
+    provider: ApiProvider;
+  }
+
+  it('should call python wrapper function with correct arguments', async () => {
+    const filePath = 'path/to/script.py';
+    const functionName = 'testFunction';
+    const context = {
+      vars: { key: 'value' },
+      provider: {
+        id: () => 'providerId',
+        label: 'providerLabel',
+        callApi: vi.fn(),
+      } as ApiProvider,
+    } as PythonContext;
+
+    const mockRunPython = vi.mocked(runPython);
+    mockRunPython.mockResolvedValue('mocked result');
+
+    await expect(pythonPromptFunction(filePath, functionName, context)).resolves.toBe(
+      'mocked result',
+    );
+    expect(mockRunPython).toHaveBeenCalledWith(filePath, functionName, [
+      {
+        ...context,
+        provider: {
+          id: 'providerId',
+          label: 'providerLabel',
+        },
+        config: {},
+      },
+    ]);
+  });
+
+  it('should call legacy function with correct arguments', async () => {
+    const filePath = 'path/to/script.py';
+    const context = {
+      vars: { key: 'value' },
+      provider: { id: () => 'providerId', label: 'providerLabel' } as ApiProvider,
+    } as PythonContext;
+    const mockPythonShellRun = vi.mocked(PythonShell.run);
+    const mockLoggerDebug = vi.mocked(logger.debug);
+    mockPythonShellRun.mockImplementation(() => {
+      return Promise.resolve(['mocked result']);
+    });
+    await expect(pythonPromptFunctionLegacy(filePath, context)).resolves.toBe('mocked result');
+    expect(mockPythonShellRun).toHaveBeenCalledWith(filePath, {
+      mode: 'text',
+      pythonPath: process.env.PROMPTFOO_PYTHON || 'python',
+      args: [
+        JSON.stringify({
+          vars: context.vars,
+          provider: {
+            id: context.provider.id(),
+            label: context.provider.label,
+          },
+          config: {},
+        }),
+      ],
+    });
+    expect(mockLoggerDebug).toHaveBeenCalledWith(`Executing python prompt script ${filePath}`);
+    expect(mockLoggerDebug).toHaveBeenCalledWith(
+      `Python prompt script ${filePath} returned: mocked result`,
+    );
+  });
+});

test/prompts/processors/string.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest';
+import { processString } from '../../../src/prompts/processors/string';
+
+import type { Prompt } from '../../../src/types/index';
+
+describe('processString', () => {
+  it('should process a valid string prompt without a label', () => {
+    const prompt: Partial<Prompt> = { raw: 'This is a prompt' };
+    expect(processString(prompt)).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'This is a prompt',
+      },
+    ]);
+  });
+
+  it('should process a valid string prompt with a label', () => {
+    const prompt: Partial<Prompt> = { raw: 'This is a prompt', label: 'Label' };
+    expect(processString(prompt)).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'Label',
+      },
+    ]);
+  });
+});

test/prompts/processors/text.test.ts

@@ -0,0 +1,107 @@
+import * as fs from 'fs';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { PROMPT_DELIMITER } from '../../../src/prompts/constants';
+import { processTxtFile } from '../../../src/prompts/processors/text';
+
+vi.mock('fs');
+
+describe('processTxtFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should process a text file with single prompt and no label', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'This is a prompt';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'file.txt: This is a prompt',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a text file with single prompt and a label', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'This is a prompt';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, { label: 'prompt 1' })).toEqual([
+      {
+        raw: 'This is a prompt',
+        label: 'prompt 1: file.txt: This is a prompt',
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should process a text file with multiple prompts and a label', () => {
+    const fileContent = `Prompt 1\n${PROMPT_DELIMITER}\nPrompt 2\n${PROMPT_DELIMITER}\nPrompt 3`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile('file.txt', { label: 'Label' })).toEqual([
+      {
+        raw: 'Prompt 1',
+        label: `Label: file.txt: Prompt 1`,
+      },
+      {
+        raw: 'Prompt 2',
+        label: `Label: file.txt: Prompt 2`,
+      },
+      {
+        raw: 'Prompt 3',
+        label: `Label: file.txt: Prompt 3`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith('file.txt', 'utf-8');
+  });
+
+  it('should handle text file with leading and trailing delimiters', () => {
+    const filePath = 'file.txt';
+    const fileContent = `${PROMPT_DELIMITER}\nPrompt 1\n${PROMPT_DELIMITER}\nPrompt 2\n${PROMPT_DELIMITER}`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'Prompt 1',
+        label: `${filePath}: Prompt 1`,
+      },
+      {
+        raw: 'Prompt 2',
+        label: `${filePath}: Prompt 2`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should return an empty array for a file with only delimiters', () => {
+    const filePath = 'file.txt';
+    const fileContent = `${PROMPT_DELIMITER}\n${PROMPT_DELIMITER}`;
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should return an empty array for an empty file', () => {
+    const filePath = 'file.txt';
+    const fileContent = '';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+
+  it('should not split on lines that contain repeated hyphens', () => {
+    const filePath = 'file.txt';
+    const fileContent = 'Line 1\n-----------------------------------\nLine 2';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processTxtFile(filePath, {})).toEqual([
+      {
+        raw: 'Line 1\n-----------------------------------\nLine 2',
+        label: `${filePath}: Line 1\n-----------------------------------\nLine 2`,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf-8');
+  });
+});

test/prompts/processors/yaml.test.ts

@@ -0,0 +1,285 @@
+import * as fs from 'fs';
+
+import dedent from 'dedent';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import { processYamlFile } from '../../../src/prompts/processors/yaml';
+import * as fileModule from '../../../src/util/file';
+
+vi.mock('fs');
+vi.mock('../../../src/util/file');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+  },
+}));
+
+describe('processYamlFile', () => {
+  const mockReadFileSync = vi.mocked(fs.readFileSync);
+  const mockMaybeLoadConfigFromExternalFile = vi.mocked(fileModule.maybeLoadConfigFromExternalFile);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // By default, maybeLoadConfigFromExternalFile returns its input unchanged
+    mockMaybeLoadConfigFromExternalFile.mockImplementation((input) => input);
+  });
+
+  it('should process a valid YAML file without a label', () => {
+    const filePath = 'file.yaml';
+    const fileContent = 'key: value';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processYamlFile(filePath, {})).toEqual([
+      {
+        raw: JSON.stringify({ key: 'value' }),
+        label: `${filePath}: ${JSON.stringify({ key: 'value' })}`,
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should process a valid YAML file with a label', () => {
+    const filePath = 'file.yaml';
+    const fileContent = 'key: value';
+    mockReadFileSync.mockReturnValue(fileContent);
+    expect(processYamlFile(filePath, { label: 'Label' })).toEqual([
+      {
+        raw: JSON.stringify({ key: 'value' }),
+        label: 'Label',
+        config: undefined,
+      },
+    ]);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should throw an error if the file cannot be read', () => {
+    const filePath = 'nonexistent.yaml';
+    mockReadFileSync.mockImplementation(() => {
+      throw new Error('File not found');
+    });
+
+    expect(() => processYamlFile(filePath, {})).toThrow('File not found');
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should parse YAML and return stringified JSON', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+key1: value1
+key2: value2
+    `;
+    const expectedJson = JSON.stringify({ key1: 'value1', key2: 'value2' });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+    expect(mockReadFileSync).toHaveBeenCalledWith(filePath, 'utf8');
+  });
+
+  it('should handle YAML with nested structures', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+parent:
+  child1: value1
+  child2: value2
+array:
+  - item1
+  - item2
+    `;
+    const expectedJson = JSON.stringify({
+      parent: { child1: 'value1', child2: 'value2' },
+      array: ['item1', 'item2'],
+    });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+  });
+
+  it('should handle YAML with whitespace in values', () => {
+    const filePath = 'file.yaml';
+    const fileContent = `
+key: "value with    spaces"
+template: "{{ variable }}   "
+    `;
+    const expectedJson = JSON.stringify({
+      key: 'value with    spaces',
+      template: '{{ variable }}   ',
+    });
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    const result = processYamlFile(filePath, {});
+    expect(result[0].raw).toBe(expectedJson);
+  });
+
+  it('should handle invalid YAML and return raw file contents', () => {
+    const filePath = 'issue-2368.yaml';
+    const fileContent = dedent`
+    {% import "system_prompt.yaml" as system_prompt %}
+    {% import "user_prompt.yaml" as user_prompt %}
+    {{ system_prompt.system_prompt() }}
+    {{ user_prompt.user_prompt(example) }}`;
+
+    mockReadFileSync.mockReturnValue(fileContent);
+
+    expect(processYamlFile(filePath, {})).toEqual([
+      {
+        raw: fileContent,
+        label: `${filePath}: ${fileContent.slice(0, 80)}`,
+        config: undefined,
+      },
+    ]);
+    expect(logger.debug).toHaveBeenCalledWith(
+      expect.stringMatching(/Error parsing YAML file issue-2368\.yaml:/),
+    );
+  });
+
+  describe('recursive file:// resolution', () => {
+    it('should recursively resolve file:// references in YAML content', () => {
+      const filePath = 'conversation.yaml';
+      const fileContent = dedent`
+        - role: system
+          content: file://system.md
+        - role: user
+          content: file://user.md
+      `;
+
+      const parsedYaml = [
+        { role: 'system', content: 'file://system.md' },
+        { role: 'user', content: 'file://user.md' },
+      ];
+
+      const resolvedContent = [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'What is 2 + 2?' },
+      ];
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle nested file:// references in complex structures', () => {
+      const filePath = 'config.yaml';
+      const fileContent = dedent`
+        provider:
+          id: openai:gpt-4
+          config:
+            temperature: file://temperature.json
+            tools: file://tools.yaml
+        messages:
+          - role: system
+            content: file://system.txt
+      `;
+
+      const parsedYaml = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 'file://temperature.json',
+            tools: 'file://tools.yaml',
+          },
+        },
+        messages: [{ role: 'system', content: 'file://system.txt' }],
+      };
+
+      const resolvedContent = {
+        provider: {
+          id: 'openai:gpt-4',
+          config: {
+            temperature: 0.7,
+            tools: [{ name: 'search', description: 'Search the web' }],
+          },
+        },
+        messages: [{ role: 'system', content: 'You are a helpful assistant.' }],
+      };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(resolvedContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(resolvedContent));
+    });
+
+    it('should handle when maybeLoadConfigFromExternalFile returns unchanged content', () => {
+      const filePath = 'no-refs.yaml';
+      const fileContent = dedent`
+        key1: value1
+        key2: value2
+      `;
+
+      const parsedYaml = { key1: 'value1', key2: 'value2' };
+
+      mockReadFileSync.mockReturnValue(fileContent);
+      mockMaybeLoadConfigFromExternalFile.mockReturnValue(parsedYaml);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(mockMaybeLoadConfigFromExternalFile).toHaveBeenCalledWith(parsedYaml);
+      expect(result[0].raw).toBe(JSON.stringify(parsedYaml));
+    });
+
+    it('should preserve empty string values when parsing YAML', () => {
+      const filePath = 'empty-value.yaml';
+      const fileContent = dedent`
+        prompts:
+          - key: ""
+      `;
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      const parsed = JSON.parse(result[0].raw as string);
+      expect(parsed).toEqual({ prompts: [{ key: '' }] });
+    });
+
+    it('should preserve null values when parsing YAML', () => {
+      const filePath = 'null-value.yaml';
+      const fileContent = dedent`
+        prompts:
+          - key: null
+        options:
+          temperature: ~
+      `;
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      const parsed = JSON.parse(result[0].raw as string);
+      expect(parsed).toEqual({
+        prompts: [{ key: null }],
+        options: { temperature: null },
+      });
+    });
+
+    it('should handle empty YAML document', () => {
+      const filePath = 'empty.yaml';
+      const fileContent = '';
+
+      mockReadFileSync.mockReturnValue(fileContent);
+
+      const result = processYamlFile(filePath, {});
+
+      expect(result).toHaveLength(1);
+      // Empty YAML parses to undefined; maybeLoadConfigFromExternalFile passes it
+      // through; JSON.stringify(undefined) is the literal string "undefined".
+      expect(result[0].raw).toBe(JSON.stringify(undefined));
+    });
+  });
+});