chore: import upstream snapshot with attribution

test/python/fixtures/test_embeddings_only.py

@@ -0,0 +1,18 @@
+"""
+Test fixture for embeddings-only provider.
+This file intentionally does NOT have call_api - only call_embedding_api.
+This simulates the user's scenario from #6072.
+"""
+
+
+def call_embedding_api(prompt, options):
+    """Only embedding functionality - no call_api defined."""
+    # Simple embedding simulation
+    words = prompt.lower().split()
+    # Create a simple embedding based on word count and first letter
+    embedding = [
+        len(words) * 0.1,
+        ord(words[0][0]) * 0.01 if words else 0.0,
+        len(prompt) * 0.01,
+    ]
+    return {"embedding": embedding}

test/python/fixtures/test_nonexistent_function.py

@@ -0,0 +1,9 @@
+"""
+Test fixture for testing error when calling a non-existent function.
+This file intentionally has no call_nonexistent_api function.
+"""
+
+
+def call_api(prompt, options, context):
+    """Regular call_api function that exists."""
+    return {"output": "This function exists"}

test/python/fixtures/test_wrong_function_name.py

@@ -0,0 +1,19 @@
+"""
+Test fixture simulating user error: using 'get_' prefix instead of 'call_' prefix.
+This mimics the issue reported in #6072 where user had get_embedding_api instead of call_embedding_api.
+"""
+
+
+def call_api(prompt, options, context):
+    """Valid default function for initialization."""
+    return {"output": "test"}
+
+
+def get_embedding_api(prompt, options):
+    """User mistakenly named this 'get_embedding_api' instead of 'call_embedding_api'."""
+    return {"embedding": [0.1, 0.2, 0.3]}
+
+
+def some_helper_function():
+    """Just another function to show in the available functions list."""
+    return "helper"

test/python/performance.bench.ts

@@ -0,0 +1,51 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { describe, expect, it } from 'vitest';
+import { PythonProvider } from '../../src/providers/pythonCompletion';
+
+describe('Performance Benchmarks (manual)', () => {
+  function writeTempPython(content: string): string {
+    const tempFile = path.join(os.tmpdir(), `bench-${Date.now()}.py`);
+    fs.writeFileSync(tempFile, content);
+    return tempFile;
+  }
+
+  it('benchmark: heavy import speedup', async () => {
+    const scriptPath = writeTempPython(`
+import time
+time.sleep(1)  # Simulate 1s import
+
+def call_api(prompt, options, context):
+    return {"output": "test"}
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: { basePath: process.cwd() },
+    });
+
+    const start = Date.now();
+    await provider.initialize();
+    const initTime = Date.now() - start;
+
+    console.log(`Initialization (1 worker): ${initTime}ms`);
+
+    const callStart = Date.now();
+    for (let i = 0; i < 10; i++) {
+      await provider.callApi(`test ${i}`);
+    }
+    const totalCallTime = Date.now() - callStart;
+    const avgCallTime = totalCallTime / 10;
+
+    console.log(`10 calls total: ${totalCallTime}ms`);
+    console.log(`Avg per call: ${avgCallTime}ms`);
+    console.log(`Expected without persistence: ~10,000ms (1s import × 10 calls)`);
+    console.log(`Speedup: ${(10000 / totalCallTime).toFixed(1)}x`);
+
+    expect(avgCallTime).toBeLessThan(100); // Each call should be fast
+
+    await provider.shutdown();
+    fs.unlinkSync(scriptPath);
+  }, 30000);
+});

test/python/python.integration.test.ts

@@ -0,0 +1,204 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+import { PythonProvider } from '../../src/providers/pythonCompletion';
+
+describe('Python Provider Integration Tests', () => {
+  let tempFiles: string[] = [];
+
+  afterEach(async () => {
+    // Cleanup temp files
+    tempFiles.forEach((file) => {
+      try {
+        fs.unlinkSync(file);
+      } catch (_e) {
+        // ignore
+      }
+    });
+    tempFiles = [];
+  });
+
+  function writeTempPython(content: string): string {
+    const tempFile = path.join(
+      os.tmpdir(),
+      `test-provider-${Date.now()}-${Math.random().toString(16).slice(2)}.py`,
+    );
+    fs.writeFileSync(tempFile, content);
+    tempFiles.push(tempFile);
+    return tempFile;
+  }
+
+  it('should handle heavy imports efficiently (load once)', async () => {
+    const scriptPath = writeTempPython(`
+import time
+
+# Simulate heavy import
+print("Loading heavy library...", flush=True)
+time.sleep(0.5)  # 500ms "import" time
+print("Library loaded!", flush=True)
+
+load_time = time.time()
+
+def call_api(prompt, options, context):
+    return {
+        "output": f"Loaded at: {load_time}",
+        "load_time": load_time
+    }
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: { basePath: process.cwd() },
+    });
+
+    await provider.initialize();
+
+    // Three calls should all reuse the same loaded module
+    const result1 = await provider.callApi('test1');
+    const result2 = await provider.callApi('test2');
+    const result3 = await provider.callApi('test3');
+
+    // Verify same load_time across all calls (same process, no re-import)
+    expect(result1.output).toContain('Loaded at:');
+    expect(result1.output).toBe(result2.output);
+    expect(result2.output).toBe(result3.output);
+
+    await provider.shutdown();
+  }, 10000);
+
+  it('should handle Unicode correctly (cross-platform)', async () => {
+    const scriptPath = writeTempPython(`
+def call_api(prompt, options, context):
+    return {
+        "output": f"Echo: {prompt}",
+        "emoji": "🚀",
+        "cjk": "你好世界",
+        "accents": "Café, naïve, Ångström"
+    }
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: { basePath: process.cwd() },
+    });
+
+    await provider.initialize();
+
+    const result = await provider.callApi('Test with emoji: 😀 and CJK: 測試');
+
+    expect(result.output).toContain('😀');
+    expect(result.output).toContain('測試');
+    expect(result.output).toBe('Echo: Test with emoji: 😀 and CJK: 測試');
+    expect((result as any).emoji).toBe('🚀');
+    expect((result as any).cjk).toBe('你好世界');
+    expect((result as any).accents).toContain('Café');
+
+    await provider.shutdown();
+  });
+
+  it('should handle async Python functions', async () => {
+    const scriptPath = writeTempPython(`
+import asyncio
+
+async def call_api(prompt, options, context):
+    await asyncio.sleep(0.1)
+    return {"output": f"Async: {prompt}"}
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: { basePath: process.cwd() },
+    });
+
+    await provider.initialize();
+    const result = await provider.callApi('async test');
+
+    expect(result.output).toBe('Async: async test');
+
+    await provider.shutdown();
+  });
+
+  it('should handle errors gracefully without crashing worker', async () => {
+    const scriptPath = writeTempPython(`
+def call_api(prompt, options, context):
+    if "error" in prompt:
+        raise ValueError("Intentional error")
+    return {"output": f"OK: {prompt}"}
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: { basePath: process.cwd() },
+    });
+
+    await provider.initialize();
+
+    // First call succeeds
+    const result1 = await provider.callApi('good');
+    expect(result1.output).toBe('OK: good');
+
+    // Second call errors
+    await expect(provider.callApi('error here')).rejects.toThrow('Intentional error');
+
+    // Third call succeeds (worker still alive!)
+    const result3 = await provider.callApi('good again');
+    expect(result3.output).toBe('OK: good again');
+
+    await provider.shutdown();
+  });
+
+  it('should work with multiple workers (concurrency)', async () => {
+    const scriptPath = writeTempPython(`
+import time
+
+counter = 0
+
+def call_api(prompt, options, context):
+    global counter
+    counter += 1
+    start_time = time.time()
+    time.sleep(0.1)  # 100ms
+    end_time = time.time()
+    return {
+        "output": f"Worker count: {counter}",
+        "start_time": start_time,
+        "end_time": end_time
+    }
+`);
+
+    const provider = new PythonProvider(`file://${scriptPath}`, {
+      config: {
+        basePath: process.cwd(),
+        workers: 4, // 4 workers
+      },
+    });
+
+    await provider.initialize();
+
+    // 4 concurrent calls with 4 workers should run in parallel
+    const results = await Promise.all([
+      provider.callApi('1'),
+      provider.callApi('2'),
+      provider.callApi('3'),
+      provider.callApi('4'),
+    ]);
+
+    // Extract timestamps from results
+    const startTimes = results.map((r) => (r as any).start_time as number);
+    const endTimes = results.map((r) => (r as any).end_time as number);
+
+    // Verify parallelization by checking execution overlap:
+    // If parallel: the last call to start begins before the first call ends
+    // If sequential: each call starts after the previous ends (no overlap)
+    const maxStartTime = Math.max(...startTimes);
+    const minEndTime = Math.min(...endTimes);
+
+    // For true parallel execution, there must be overlap
+    expect(maxStartTime).toBeLessThan(minEndTime);
+
+    // Each worker maintains its own counter
+    results.forEach((r) => {
+      expect(r.output).toMatch(/Worker count: \d+/);
+    });
+
+    await provider.shutdown();
+  }, 10000);
+});

test/python/pythonUtils.test.ts

@@ -0,0 +1,673 @@
+import fs from 'fs';
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Create mock for execFileAsync - must be hoisted for vi.mock factory
+const { mockExecFileAsync, mockExecFile } = vi.hoisted(() => {
+  const mockExecFileAsync = vi.fn();
+  // Create a mock execFile with the custom promisify symbol
+  const mockExecFile = Object.assign(vi.fn(), {
+    [Symbol.for('nodejs.util.promisify.custom')]: mockExecFileAsync,
+  });
+  return { mockExecFileAsync, mockExecFile };
+});
+
+// Mock child_process.execFile with custom promisify support
+vi.mock('child_process', () => ({
+  execFile: mockExecFile,
+}));
+
+import { PythonShell } from 'python-shell';
+import { getEnvBool, getEnvString } from '../../src/envars';
+import logger from '../../src/logger';
+import * as pythonUtils from '../../src/python/pythonUtils';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../../src/util/secureTempFiles';
+
+const fsMock = vi.hoisted(() => ({
+  writeFileSync: vi.fn(),
+  readFileSync: vi.fn(),
+  unlinkSync: vi.fn(),
+}));
+
+// Mock setup
+vi.mock('fs', () => {
+  return {
+    ...fsMock,
+    default: fsMock,
+  };
+});
+
+vi.mock('fs/promises', () => ({
+  default: {
+    writeFile: fsMock.writeFileSync,
+    readFile: fsMock.readFileSync,
+    unlink: fsMock.unlinkSync,
+  },
+  writeFile: fsMock.writeFileSync,
+  readFile: fsMock.readFileSync,
+  unlink: fsMock.unlinkSync,
+}));
+
+vi.mock('../../src/envars', () => ({
+  getEnvString: vi.fn(),
+  getEnvBool: vi.fn(),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+  },
+}));
+
+vi.mock('../../src/util/secureTempFiles', () => ({
+  createSecureTempDirectory: vi.fn(),
+  removeSecureTempDirectory: vi.fn(),
+  writeSecureTempFile: vi.fn(),
+}));
+
+// Must be hoisted for vi.mock factory
+const { mockPythonShellInstance, MockPythonShell } = vi.hoisted(() => {
+  const instance = {
+    stdout: { on: vi.fn() },
+    stderr: { on: vi.fn() },
+    end: vi.fn(),
+  };
+  // Create a proper class that can be used with 'new'
+  const MockPythonShell = vi.fn(function (this: typeof instance) {
+    Object.assign(this, instance);
+    return this;
+  }) as unknown as typeof import('python-shell').PythonShell;
+  return { mockPythonShellInstance: instance, MockPythonShell };
+});
+
+vi.mock('python-shell', () => ({
+  PythonShell: MockPythonShell,
+}));
+
+describe('Python Utils', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockExecFileAsync.mockReset();
+    pythonUtils.state.cachedPythonPath = null;
+    pythonUtils.state.validationPromise = null;
+    mockPythonShellInstance.stdout.on.mockReset();
+    mockPythonShellInstance.stderr.on.mockReset();
+    mockPythonShellInstance.end.mockReset();
+    // Set default mock return values
+    vi.mocked(getEnvString).mockReturnValue('');
+    vi.mocked(getEnvBool).mockReturnValue(false);
+    vi.mocked(createSecureTempDirectory).mockResolvedValue('/tmp/promptfoo-python-test');
+    vi.mocked(removeSecureTempDirectory).mockResolvedValue(undefined);
+    vi.mocked(writeSecureTempFile).mockImplementation(
+      async (directory: string, filename: string) => `${directory}/${filename}`,
+    );
+  });
+
+  describe('getConfiguredPythonPath', () => {
+    it('should return explicit config path when provided', () => {
+      const result = pythonUtils.getConfiguredPythonPath('/custom/python/path');
+      expect(result).toBe('/custom/python/path');
+    });
+
+    it('should return PROMPTFOO_PYTHON when config path is not provided', () => {
+      vi.mocked(getEnvString).mockReturnValue('/env/python/path');
+
+      const result = pythonUtils.getConfiguredPythonPath(undefined);
+
+      expect(result).toBe('/env/python/path');
+      expect(getEnvString).toHaveBeenCalledWith('PROMPTFOO_PYTHON');
+    });
+
+    it('should prioritize config path over PROMPTFOO_PYTHON', () => {
+      vi.mocked(getEnvString).mockReturnValue('/env/python/path');
+
+      const result = pythonUtils.getConfiguredPythonPath('/config/python/path');
+
+      expect(result).toBe('/config/python/path');
+    });
+
+    it('should return undefined when neither config nor env var is set', () => {
+      vi.mocked(getEnvString).mockReturnValue('');
+
+      const result = pythonUtils.getConfiguredPythonPath(undefined);
+
+      expect(result).toBeUndefined();
+    });
+
+    it('should return undefined when config is empty string and env var is not set', () => {
+      vi.mocked(getEnvString).mockReturnValue('');
+
+      const result = pythonUtils.getConfiguredPythonPath('');
+
+      expect(result).toBeUndefined();
+    });
+
+    it('should return PROMPTFOO_PYTHON when config is empty string', () => {
+      vi.mocked(getEnvString).mockReturnValue('/env/python');
+
+      const result = pythonUtils.getConfiguredPythonPath('');
+
+      expect(result).toBe('/env/python');
+    });
+  });
+
+  describe('getSysExecutable', () => {
+    it('should return Python executable path from sys.executable', async () => {
+      // Mock for Unix-like systems (not Windows)
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'linux' });
+
+      mockExecFileAsync.mockResolvedValue({ stdout: '/usr/bin/python3.8\n', stderr: '' });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBe('/usr/bin/python3.8');
+      expect(mockExecFileAsync).toHaveBeenCalledWith('python3', [
+        '-c',
+        'import sys; print(sys.executable)',
+      ]);
+
+      // Restore original platform
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should use Windows where command first on Windows', async () => {
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'win32' });
+
+      let callCount = 0;
+      mockExecFileAsync.mockImplementation(async () => {
+        callCount++;
+        if (callCount === 1) {
+          // 'where python' returns multiple paths
+          return {
+            stdout:
+              'C:\\Users\\test\\AppData\\Local\\Microsoft\\WindowsApps\\python.exe\nC:\\Python39\\python.exe\n',
+            stderr: '',
+          };
+        } else {
+          // Version check succeeds
+          return { stdout: 'Python 3.9.0\n', stderr: '' };
+        }
+      });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      // Should skip WindowsApps and use the real Python installation
+      expect(result).toBe('C:\\Python39\\python.exe');
+      expect(mockExecFileAsync).toHaveBeenCalledWith('where', ['python']);
+      // Verify that the non-WindowsApps path was validated
+      expect(mockExecFileAsync).toHaveBeenCalledWith('C:\\Python39\\python.exe', ['--version']);
+
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should fall back to py commands if Windows where fails', async () => {
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'win32' });
+
+      let callCount = 0;
+      mockExecFileAsync.mockImplementation(async () => {
+        callCount++;
+        if (callCount === 1) {
+          // 'where python' fails
+          throw new Error('where command failed');
+        } else {
+          // 'py' sys.executable succeeds
+          return { stdout: 'C:\\Python39\\python.exe\n', stderr: '' };
+        }
+      });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBe('C:\\Python39\\python.exe');
+      expect(mockExecFileAsync).toHaveBeenCalledWith('where', ['python']);
+      // Verify py launcher fallback was used
+      expect(mockExecFileAsync).toHaveBeenCalledWith('py', [
+        '-c',
+        'import sys; print(sys.executable)',
+      ]);
+
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should try direct python command as final Windows fallback', async () => {
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'win32' });
+
+      let callCount = 0;
+      mockExecFileAsync.mockImplementation(async () => {
+        callCount++;
+        if (callCount <= 3) {
+          // First 3 calls fail
+          throw new Error('failed');
+        } else {
+          // Final fallback succeeds
+          return { stdout: 'Python 3.9.0\n', stderr: '' };
+        }
+      });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBe('python');
+      expect(mockExecFileAsync).toHaveBeenCalledWith('where', ['python']);
+      // Verify the final fallback python --version was called
+      expect(mockExecFileAsync).toHaveBeenCalledWith('python', ['--version']);
+
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should add .exe suffix on Windows if missing', async () => {
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'win32' });
+
+      let callCount = 0;
+      mockExecFileAsync.mockImplementation(async () => {
+        callCount++;
+        if (callCount === 1) {
+          throw new Error('where failed');
+        } else {
+          return { stdout: 'C:\\Python39\\python\n', stderr: '' };
+        }
+      });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBe('C:\\Python39\\python.exe');
+
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should handle empty where output gracefully', async () => {
+      const originalPlatform = process.platform;
+      Object.defineProperty(process, 'platform', { value: 'win32' });
+
+      let callCount = 0;
+      mockExecFileAsync.mockImplementation(async () => {
+        callCount++;
+        if (callCount === 1) {
+          // 'where python' returns empty
+          return { stdout: '', stderr: '' };
+        } else {
+          // 'py' succeeds
+          return { stdout: 'C:\\Python39\\python.exe\n', stderr: '' };
+        }
+      });
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBe('C:\\Python39\\python.exe');
+      expect(mockExecFileAsync).toHaveBeenCalledWith('where', ['python']);
+      // Verify py launcher fallback was used when where returned empty
+      expect(mockExecFileAsync).toHaveBeenCalledWith('py', [
+        '-c',
+        'import sys; print(sys.executable)',
+      ]);
+
+      Object.defineProperty(process, 'platform', { value: originalPlatform });
+    });
+
+    it('should return null if no Python executable is found', async () => {
+      mockExecFileAsync.mockRejectedValue(new Error('Command failed'));
+
+      const result = await pythonUtils.getSysExecutable();
+
+      expect(result).toBeNull();
+    });
+  });
+
+  describe('tryPath', () => {
+    describe('successful path validation', () => {
+      it('should return the path for a valid Python 3 executable', async () => {
+        mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+        const result = await pythonUtils.tryPath('/usr/bin/python3');
+
+        expect(result).toBe('/usr/bin/python3');
+        expect(mockExecFileAsync).toHaveBeenCalledWith('/usr/bin/python3', ['--version']);
+      });
+    });
+
+    describe('failed path validation', () => {
+      it('should return null for a non-existent executable', async () => {
+        mockExecFileAsync.mockRejectedValue(new Error('Command failed'));
+
+        const result = await pythonUtils.tryPath('/usr/bin/nonexistent');
+
+        expect(result).toBeNull();
+        expect(mockExecFileAsync).toHaveBeenCalledWith('/usr/bin/nonexistent', ['--version']);
+      });
+
+      it('should return null if the command times out', async () => {
+        vi.useFakeTimers();
+
+        // Mock execFileAsync to return a promise that never resolves (simulating timeout)
+        mockExecFileAsync.mockImplementation(() => new Promise(() => {}));
+
+        const resultPromise = pythonUtils.tryPath('/usr/bin/python3');
+        await vi.advanceTimersByTimeAsync(2501);
+
+        const result = await resultPromise;
+
+        expect(result).toBeNull();
+        expect(mockExecFileAsync).toHaveBeenCalledWith('/usr/bin/python3', ['--version']);
+        vi.useRealTimers();
+      });
+    });
+  });
+
+  describe('validatePythonPath', () => {
+    describe('caching behavior', () => {
+      it('should validate and cache an existing Python 3 path', async () => {
+        mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+        const result = await pythonUtils.validatePythonPath('python', false);
+
+        expect(result).toBe('python');
+        expect(pythonUtils.state.cachedPythonPath).toBe('python');
+        expect(mockExecFileAsync).toHaveBeenCalledWith('python', ['--version']);
+      });
+
+      it('should return the cached path on subsequent calls', async () => {
+        pythonUtils.state.cachedPythonPath = '/usr/bin/python3';
+
+        const result = await pythonUtils.validatePythonPath('python', false);
+
+        expect(result).toBe('/usr/bin/python3');
+        expect(mockExecFileAsync).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('fallback behavior', () => {
+      it('should fall back to alternative paths for non-existent programs when not explicit', async () => {
+        mockExecFileAsync.mockReset();
+        let callCount = 0;
+
+        mockExecFileAsync.mockImplementation(async () => {
+          callCount++;
+          if (callCount === 1) {
+            // Primary path fails
+            throw new Error('Command failed');
+          } else if (process.platform === 'win32') {
+            if (callCount <= 4) {
+              throw new Error('Command failed');
+            } else {
+              return { stdout: 'Python 3.9.5\n', stderr: '' };
+            }
+          } else {
+            // Unix: getSysExecutable succeeds on python3
+            return { stdout: '/usr/bin/python3\n', stderr: '' };
+          }
+        });
+
+        const result = await pythonUtils.validatePythonPath('non_existent_program', false);
+
+        expect(result).toBe(process.platform === 'win32' ? 'python' : '/usr/bin/python3');
+      });
+
+      it('should throw an error for non-existent programs when explicit', async () => {
+        mockExecFileAsync.mockRejectedValue(new Error('Command failed'));
+
+        await expect(pythonUtils.validatePythonPath('non_existent_program', true)).rejects.toThrow(
+          'Python 3 not found. Tried "non_existent_program"',
+        );
+        expect(mockExecFileAsync).toHaveBeenCalledWith('non_existent_program', ['--version']);
+      });
+
+      it('should throw an error when no valid Python path is found', async () => {
+        mockExecFileAsync.mockReset();
+        mockExecFileAsync.mockRejectedValue(new Error('Command failed'));
+
+        await expect(pythonUtils.validatePythonPath('python', false)).rejects.toThrow(
+          'Python 3 not found. Tried "python", sys.executable detection, and fallback commands.',
+        );
+        expect(mockExecFileAsync).toHaveBeenCalled();
+      });
+    });
+
+    describe('environment variable handling', () => {
+      it('should use PROMPTFOO_PYTHON environment variable when provided', async () => {
+        vi.mocked(getEnvString).mockReturnValue('/custom/python/path');
+        mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+        const result = await pythonUtils.validatePythonPath('/custom/python/path', true);
+
+        expect(result).toBe('/custom/python/path');
+        expect(mockExecFileAsync).toHaveBeenCalledWith('/custom/python/path', ['--version']);
+      });
+    });
+
+    describe('concurrent validation', () => {
+      it('should share validation promise between concurrent calls', async () => {
+        mockExecFileAsync.mockImplementation(async () => {
+          // Yield control so concurrent callers can register on the shared
+          // validation promise before resolution.
+          await Promise.resolve();
+          return { stdout: 'Python 3.8.10\n', stderr: '' };
+        });
+
+        // Start two validations concurrently
+        const [result1, result2] = await Promise.all([
+          pythonUtils.validatePythonPath('python', false),
+          pythonUtils.validatePythonPath('python', false),
+        ]);
+
+        expect(result1).toBe('python');
+        expect(result2).toBe('python');
+
+        // Only one exec call should be made
+        expect(mockExecFileAsync).toHaveBeenCalledTimes(1);
+
+        // After resolution, validation promise should be cleared
+        expect(pythonUtils.state.validationPromise).toBeNull();
+      });
+
+      it('should handle race conditions between concurrent validation attempts', async () => {
+        // Clear cached path first to ensure validation runs
+        pythonUtils.state.cachedPythonPath = null;
+
+        mockExecFileAsync.mockImplementation(async () => {
+          // Yield control so concurrent callers can race onto the shared
+          // validation promise before resolution.
+          await Promise.resolve();
+          return { stdout: 'Python 3.8.10\n', stderr: '' };
+        });
+
+        // Start multiple validations without waiting
+        const promises = [
+          pythonUtils.validatePythonPath('python', false),
+          pythonUtils.validatePythonPath('python', false),
+          pythonUtils.validatePythonPath('python', false),
+        ];
+
+        const results = await Promise.all(promises);
+
+        expect(results).toEqual(['python', 'python', 'python']);
+
+        // Only one exec call should be made
+        expect(mockExecFileAsync).toHaveBeenCalledTimes(1);
+
+        // After resolution, validation promise should be cleared
+        expect(pythonUtils.state.validationPromise).toBeNull();
+      });
+    });
+
+    describe('promise cleanup', () => {
+      it('should clear validation promise after failed validation', async () => {
+        mockExecFileAsync.mockRejectedValue(new Error('Command failed'));
+
+        await expect(pythonUtils.validatePythonPath('python', true)).rejects.toThrow(
+          'Python 3 not found. Tried "python"',
+        );
+
+        // Validation promise should be cleared even after failure
+        expect(pythonUtils.state.validationPromise).toBeNull();
+      });
+    });
+  });
+
+  describe('runPython', () => {
+    beforeEach(() => {
+      vi.clearAllMocks();
+    });
+
+    it('should execute a Python script with proper arguments', async () => {
+      vi.mocked(fs.writeFileSync).mockImplementation(() => {});
+      vi.mocked(fs.readFileSync).mockReturnValue(
+        JSON.stringify({ type: 'final_result', data: 42 }),
+      );
+      vi.mocked(fs.unlinkSync).mockImplementation(() => {});
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      const scriptPath = '/path/to/script.py';
+      const result = await pythonUtils.runPython(scriptPath, 'test_method', [1, 2, 3]);
+
+      expect(result).toBe(42);
+      expect(createSecureTempDirectory).toHaveBeenCalledWith('promptfoo-python-');
+      expect(writeSecureTempFile).toHaveBeenNthCalledWith(
+        1,
+        '/tmp/promptfoo-python-test',
+        'input.json',
+        '[1,2,3]',
+      );
+      expect(writeSecureTempFile).toHaveBeenNthCalledWith(
+        2,
+        '/tmp/promptfoo-python-test',
+        'output.json',
+        '',
+      );
+      expect(removeSecureTempDirectory).toHaveBeenCalledWith('/tmp/promptfoo-python-test');
+      expect(PythonShell).toHaveBeenCalledWith(
+        'wrapper.py',
+        expect.objectContaining({
+          scriptPath: expect.any(String),
+          args: expect.arrayContaining([path.resolve(scriptPath), 'test_method']),
+        }),
+      );
+    });
+
+    it('should use python subdirectory for wrapper.py scriptPath', async () => {
+      vi.mocked(fs.writeFileSync).mockImplementation(() => {});
+      vi.mocked(fs.readFileSync).mockReturnValue(
+        JSON.stringify({ type: 'final_result', data: 'test_result' }),
+      );
+      vi.mocked(fs.unlinkSync).mockImplementation(() => {});
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      const scriptPath = '/path/to/script.py';
+      await pythonUtils.runPython(scriptPath, 'test_method', ['arg1', 'arg2']);
+
+      // Verify PythonShell was called with scriptPath ending in 'python'
+      // This works for both development (src/python/) and production (dist/src/python/)
+      expect(PythonShell).toHaveBeenCalledWith(
+        'wrapper.py',
+        expect.objectContaining({
+          scriptPath: expect.stringMatching(/python$/),
+        }),
+      );
+    });
+
+    it('should throw an error if Python script returns invalid JSON', async () => {
+      vi.mocked(fs.writeFileSync).mockImplementation(() => {});
+      vi.mocked(fs.readFileSync).mockReturnValue('invalid json');
+      vi.mocked(fs.unlinkSync).mockImplementation(() => {});
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      await expect(pythonUtils.runPython('/path/to/script.py', 'test_method', [])).rejects.toThrow(
+        'Invalid JSON',
+      );
+    });
+
+    it('should not log arguments or returned payloads', async () => {
+      vi.mocked(fs.readFileSync).mockReturnValue(
+        JSON.stringify({ type: 'final_result', data: 'secret-result' }),
+      );
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      await pythonUtils.runPython('/path/to/script.py', 'test_method', ['secret-input']);
+
+      const debugMessages = vi.mocked(logger.debug).mock.calls.flat().map(String).join('\n');
+      expect(debugMessages).not.toContain('secret-input');
+      expect(debugMessages).not.toContain('secret-result');
+    });
+
+    it('classifies routine stderr without reporting Python logging as errors', async () => {
+      vi.mocked(fs.readFileSync).mockReturnValue(
+        JSON.stringify({ type: 'final_result', data: 'result' }),
+      );
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+      mockPythonShellInstance.stderr.on.mockImplementation((event: string, callback: any) => {
+        if (event === 'data') {
+          callback(Buffer.from('INFO:root:loaded config\nWARNING:root:slow response\n'));
+        }
+      });
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      await pythonUtils.runPython('/path/to/script.py', 'test_method', []);
+
+      expect(logger.info).toHaveBeenCalledWith('INFO:root:loaded config');
+      expect(logger.warn).toHaveBeenCalledWith('WARNING:root:slow response');
+      expect(logger.error).not.toHaveBeenCalled();
+    });
+
+    it('should throw an error if Python script does not return final_result', async () => {
+      vi.mocked(fs.writeFileSync).mockImplementation(() => {});
+      vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ type: 'other', data: 42 }));
+      vi.mocked(fs.unlinkSync).mockImplementation(() => {});
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      await expect(pythonUtils.runPython('/path/to/script.py', 'test_method', [])).rejects.toThrow(
+        'The Python script `call_api` function must return a dict with an `output`',
+      );
+    });
+
+    it('should clean up temporary files even on error', async () => {
+      vi.mocked(fs.writeFileSync).mockImplementation(() => {});
+      vi.mocked(fs.readFileSync).mockImplementation(() => {
+        throw new Error('Read failed');
+      });
+      vi.mocked(fs.unlinkSync).mockImplementation(() => {});
+      mockExecFileAsync.mockResolvedValue({ stdout: 'Python 3.8.10\n', stderr: '' });
+
+      mockPythonShellInstance.end.mockImplementation((callback: any) => {
+        callback(null);
+      });
+
+      await expect(
+        pythonUtils.runPython('/path/to/script.py', 'test_method', []),
+      ).rejects.toThrow();
+
+      expect(removeSecureTempDirectory).toHaveBeenCalledWith('/tmp/promptfoo-python-test');
+    });
+  });
+});

test/python/windows-path.test.ts

@@ -0,0 +1,205 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { PythonProvider } from '../../src/providers/pythonCompletion';
+import * as pythonUtils from '../../src/python/pythonUtils';
+import { mockProcessEnv } from '../util/utils';
+
+// Windows CI has severe filesystem delays - allow up to 90s
+const TEST_TIMEOUT = process.platform === 'win32' ? 90000 : 15000;
+
+// Windows-specific test to verify pipe delimiter handles drive letters correctly
+// This test ensures paths like C:\ don't break the protocol parsing
+describe('PythonProvider Windows Path Handling', () => {
+  let tempDir: string;
+  let restoreEnv: () => void;
+  let pathProvider: PythonProvider;
+  let concurrentProvider: PythonProvider;
+  let protocolProvider: PythonProvider;
+  let specialCharsProvider: PythonProvider;
+  const providers: PythonProvider[] = [];
+
+  const createProvider = (scriptName: string, scriptContent: string) => {
+    const scriptPath = path.join(tempDir, scriptName);
+    fs.writeFileSync(scriptPath, scriptContent);
+    const provider = new PythonProvider(scriptPath, {
+      id: `python:${scriptName}`,
+      config: { basePath: tempDir },
+    });
+    providers.push(provider);
+    return provider;
+  };
+
+  beforeAll(async () => {
+    // Reset Python state
+    pythonUtils.state.cachedPythonPath = null;
+    pythonUtils.state.validationPromise = null;
+
+    restoreEnv = mockProcessEnv({ PROMPTFOO_CACHE_ENABLED: 'false' });
+
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-windows-path-test-'));
+
+    pathProvider = createProvider(
+      'path_test.py',
+      `
+import os
+
+def call_api(prompt, options, context):
+    temp_dir = os.environ.get('TEMP', os.environ.get('TMP', '/tmp'))
+    return {
+        "output": f"Processed: {prompt}",
+        "metadata": {
+            "temp_path": temp_dir,
+            "has_colon": ":" in temp_dir
+        }
+    }
+`,
+    );
+
+    concurrentProvider = createProvider(
+      'concurrent_test.py',
+      `
+def call_api(prompt, options, context):
+    return {
+        "output": f"Processed: {prompt}"
+    }
+`,
+    );
+
+    protocolProvider = createProvider(
+      'protocol_test.py',
+      `
+def call_api(prompt, options, context):
+    # If we got here, the protocol parsing worked correctly
+    return {
+        "output": "Protocol parsing successful",
+        "platform": "${process.platform}"
+    }
+`,
+    );
+
+    specialCharsProvider = createProvider(
+      'special_chars.py',
+      `
+def call_api(prompt, options, context):
+    import os
+    temp_dir = os.environ.get('TEMP', '/tmp')
+    return {
+        "output": "Success",
+        "metadata": {
+            "temp_dir": temp_dir,
+            "has_special_chars": any(c in temp_dir for c in [' ', '-', '_', '.'])
+        }
+    }
+`,
+    );
+
+    await Promise.all(providers.map((provider) => provider.initialize()));
+  }, TEST_TIMEOUT);
+
+  afterAll(async () => {
+    // Cleanup providers
+    const shutdownResults = await Promise.allSettled(
+      providers.map(async (provider) => ({
+        providerId: provider.id(),
+        result: await provider.shutdown(),
+      })),
+    );
+    const shutdownFailures = shutdownResults
+      .map((result, index) =>
+        result.status === 'rejected'
+          ? `${providers[index]?.id() ?? `provider-${index}`}: ${String(result.reason)}`
+          : null,
+      )
+      .filter((failure): failure is string => failure !== null);
+
+    providers.length = 0;
+
+    if (tempDir && fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { recursive: true });
+    }
+
+    pythonUtils.state.cachedPythonPath = null;
+    pythonUtils.state.validationPromise = null;
+
+    restoreEnv();
+
+    if (shutdownFailures.length > 0) {
+      throw new Error(`PythonProvider shutdown failed: ${shutdownFailures.join('; ')}`);
+    }
+  }, TEST_TIMEOUT);
+
+  it(
+    'should handle paths with colons (like C:\\ on Windows)',
+    async () => {
+      // This test verifies that the protocol delimiter (pipe |) doesn't conflict
+      // with Windows drive letters (C:, D:, etc.) in file paths
+      const result = await pathProvider.callApi('Test prompt');
+
+      // Verify the call succeeded
+      expect(result.output).toBe('Processed: Test prompt');
+      expect(result.error).toBeUndefined();
+
+      // On Windows, temp path should contain a colon (C:, D:, etc.)
+      if (process.platform === 'win32') {
+        expect(result.metadata?.has_colon).toBe(true);
+        expect(result.metadata?.temp_path).toMatch(/^[A-Z]:\\/);
+      }
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle multiple concurrent calls with Windows paths',
+    async () => {
+      // Stress test: ensure protocol works with multiple concurrent requests
+      // where temp file paths all contain colons
+      // Execute multiple calls concurrently
+      const promises = [];
+      for (let i = 0; i < 5; i++) {
+        promises.push(concurrentProvider.callApi(`Request ${i}`));
+      }
+
+      const results = await Promise.all(promises);
+
+      // All calls should succeed
+      expect(results).toHaveLength(5);
+      results.forEach((result, index) => {
+        expect(result.error).toBeUndefined();
+        expect(result.output).toBe(`Processed: Request ${index}`);
+      });
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should parse protocol commands correctly with Windows paths',
+    async () => {
+      // This test verifies the internal protocol command parsing
+      // Command format: CALL|function_name|request_file|response_file
+      // With Windows paths: CALL|call_api|C:\path\req.json|C:\path\resp.json
+      const result = await protocolProvider.callApi('Test');
+
+      expect(result.output).toBe('Protocol parsing successful');
+      expect(result.error).toBeUndefined();
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle paths with special characters',
+    async () => {
+      // Test that paths with various special characters work
+      // (except pipe | which is the delimiter)
+      const result = await specialCharsProvider.callApi('Test');
+
+      expect(result.output).toBe('Success');
+      expect(result.error).toBeUndefined();
+      // Verify temp directory path was processed correctly
+      expect(result.metadata?.temp_dir).toBeTruthy();
+    },
+    TEST_TIMEOUT,
+  );
+});

test/python/worker.test.ts

@@ -0,0 +1,670 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import logger from '../../src/logger';
+import { MAX_STDERR_BUFFER_LENGTH, PythonWorker } from '../../src/python/worker';
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    error: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+  },
+}));
+
+afterEach(() => {
+  vi.clearAllMocks();
+});
+
+// Windows CI has severe filesystem delays (antivirus, etc.) - allow up to 90s
+// Non-Windows CI can also have timing variance with Python IPC, so use 15s (matching windows-path.test.ts)
+const TEST_TIMEOUT = process.platform === 'win32' ? 90000 : 15000;
+
+// Skip on Windows CI due to aggressive file security policies blocking temp file IPC
+// Works fine on local Windows and all other platforms
+const describeOrSkip = process.platform === 'win32' && process.env.CI ? describe.skip : describe;
+
+type TestablePythonWorker = {
+  flushStderr(): void;
+  handleDone(responseFile: string): void;
+  handleStderr(data: Buffer | string): void;
+  pendingRequest: {
+    responseFile: string;
+    resolve: (result: unknown) => void;
+    reject: (error: Error) => void;
+  } | null;
+};
+
+function createTestableWorker() {
+  return new PythonWorker(
+    path.join(os.tmpdir(), 'provider.py'),
+    'call_api',
+  ) as unknown as TestablePythonWorker;
+}
+
+describe('PythonWorker stderr parsing', () => {
+  it('honors explicit INFO and DEBUG prefixes before scanning message text', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr('INFO:loaded error budget config\nDEBUG:error retry state\n');
+
+    expect(logger.info).toHaveBeenCalledWith(
+      'Python worker stderr: INFO:loaded error budget config',
+    );
+    expect(logger.debug).toHaveBeenCalledWith('Python worker stderr: DEBUG:error retry state');
+    expect(logger.error).not.toHaveBeenCalled();
+  });
+
+  it('keeps traceback continuation lines at error level and preserves indentation', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr(
+      [
+        'ERROR: Failed to load module: boom',
+        'Traceback (most recent call last):',
+        '  File "/tmp/provider.py", line 1, in <module>',
+        '    raise ValueError("boom")',
+        'ValueError: boom',
+        '',
+      ].join('\n'),
+    );
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr: ERROR: Failed to load module: boom',
+    );
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr: Traceback (most recent call last):',
+    );
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr:   File "/tmp/provider.py", line 1, in <module>',
+    );
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr:     raise ValueError("boom")');
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: ValueError: boom');
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('classifies the standard Python logging format (LEVEL:name:message)', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr(
+      ['DEBUG:root:retry state', 'INFO:root:loaded config', 'WARNING:urllib3:pool full', ''].join(
+        '\n',
+      ),
+    );
+
+    expect(logger.debug).toHaveBeenCalledWith('Python worker stderr: DEBUG:root:retry state');
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:root:loaded config');
+    expect(logger.warn).toHaveBeenCalledWith('Python worker stderr: WARNING:urllib3:pool full');
+    expect(logger.error).not.toHaveBeenCalled();
+  });
+
+  it('ends a traceback at its exception summary without a trailing blank line', () => {
+    const worker = createTestableWorker();
+
+    // A real traceback.print_exc() emits no trailing blank line. The summary
+    // line itself must terminate the traceback so later stderr is not poisoned.
+    worker.handleStderr(
+      [
+        'Traceback (most recent call last):',
+        '  File "/tmp/provider.py", line 1, in <module>',
+        'ValueError: boom',
+        '',
+      ].join('\n'),
+    );
+
+    worker.handleStderr('routine progress from the next call\n');
+
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: ValueError: boom');
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Python worker stderr: routine progress from the next call',
+    );
+    expect(logger.error).not.toHaveBeenCalledWith(
+      'Python worker stderr: routine progress from the next call',
+    );
+  });
+
+  it('does not leak traceback state across calls in a long-lived worker', () => {
+    const worker = createTestableWorker();
+
+    // Call 1 prints a handled traceback (no trailing blank line).
+    worker.handleStderr(
+      'Traceback (most recent call last):\n  File "x", line 1\nValueError: boom\n',
+    );
+    // Call 2 emits a plain, unprefixed stderr line.
+    worker.handleStderr('plain progress line\n');
+
+    expect(logger.error).not.toHaveBeenCalledWith('Python worker stderr: plain progress line');
+    expect(logger.warn).toHaveBeenCalledWith('Python worker stderr: plain progress line');
+  });
+
+  it('keeps chained exception reports coherent at error level', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr(
+      [
+        'Traceback (most recent call last):',
+        '  File "x", line 1',
+        'ValueError: inner',
+        '',
+        'During handling of the above exception, another exception occurred:',
+        '',
+        'Traceback (most recent call last):',
+        '  File "x", line 2',
+        'RuntimeError: outer',
+        '',
+      ].join('\n'),
+    );
+
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: ValueError: inner');
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr: During handling of the above exception, another exception occurred:',
+    );
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: RuntimeError: outer');
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('does not mistake an indented traceback source line for a log prefix', () => {
+    const worker = createTestableWorker();
+
+    // The displayed source frame happens to start with the word INFO.
+    worker.handleStderr(
+      [
+        'Traceback (most recent call last):',
+        '  File "/tmp/provider.py", line 2, in call_api',
+        '    INFO = build_info()',
+        'ValueError: boom',
+        '',
+      ].join('\n'),
+    );
+
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr:     INFO = build_info()');
+    expect(logger.info).not.toHaveBeenCalled();
+  });
+
+  it('keeps a buffered final traceback line at error level on flush', () => {
+    const worker = createTestableWorker();
+
+    // Worker dies mid-write: the exception summary has no trailing newline.
+    worker.handleStderr('Traceback (most recent call last):\n  File "x", line 1\nValueError: boom');
+    worker.flushStderr();
+
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: ValueError: boom');
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('keeps traceback continuation at error level when a CRLF pair is split across chunks', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr('Traceback (most recent call last):\r');
+    worker.handleStderr('\n  File "x", line 1\r\nValueError: boom\r\n');
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr: Traceback (most recent call last):',
+    );
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr:   File "x", line 1');
+    expect(logger.error).toHaveBeenCalledWith('Python worker stderr: ValueError: boom');
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('decodes multi-byte stderr characters split across buffer chunks', () => {
+    const worker = createTestableWorker();
+    const full = Buffer.from('INFO:café\n', 'utf8');
+
+    // Split inside the two-byte UTF-8 sequence for é.
+    worker.handleStderr(full.subarray(0, full.length - 2));
+    expect(logger.info).not.toHaveBeenCalled();
+
+    worker.handleStderr(full.subarray(full.length - 2));
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:café');
+  });
+
+  it('does not treat a one-line ERROR log as traceback continuation state', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr('ERROR:provider reported a recoverable issue\nplain stderr later\n');
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Python worker stderr: ERROR:provider reported a recoverable issue',
+    );
+    expect(logger.warn).toHaveBeenCalledWith('Python worker stderr: plain stderr later');
+  });
+
+  it('buffers split stderr chunks before classifying complete lines', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr('IN');
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.info).not.toHaveBeenCalled();
+
+    worker.handleStderr('FO:loaded error budget config\n');
+
+    expect(logger.info).toHaveBeenCalledWith(
+      'Python worker stderr: INFO:loaded error budget config',
+    );
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('flushes an unterminated buffered stderr line', () => {
+    const worker = createTestableWorker();
+
+    worker.handleStderr('WARNING:partial stderr line');
+    expect(logger.warn).not.toHaveBeenCalled();
+
+    worker.flushStderr();
+
+    expect(logger.warn).toHaveBeenCalledWith('Python worker stderr: WARNING:partial stderr line');
+  });
+
+  it('treats bare carriage returns as line delimiters', () => {
+    const worker = createTestableWorker();
+
+    // Each \r that is followed by more data is a complete line ending.
+    worker.handleStderr('INFO:step 1\rINFO:step 2\rINFO:step 3\n');
+
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:step 1');
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:step 2');
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:step 3');
+  });
+
+  it('holds a trailing carriage return until the next chunk disambiguates it', () => {
+    const worker = createTestableWorker();
+
+    // A trailing \r might be the first half of a split \r\n, so it waits.
+    worker.handleStderr('INFO:buffered step\r');
+    expect(logger.info).not.toHaveBeenCalled();
+
+    worker.flushStderr();
+    expect(logger.info).toHaveBeenCalledWith('Python worker stderr: INFO:buffered step');
+  });
+
+  it('bounds an unterminated stderr buffer', () => {
+    const worker = createTestableWorker();
+    const longLine = 'x'.repeat(MAX_STDERR_BUFFER_LENGTH);
+
+    worker.handleStderr(longLine);
+
+    expect(logger.warn).toHaveBeenCalledWith(`Python worker stderr: ${longLine}`);
+  });
+});
+
+describe('PythonWorker completion markers', () => {
+  it('accepts a valid response path terminated by a carriage return', () => {
+    const worker = createTestableWorker();
+    const responseFile = path.join(os.tmpdir(), 'response.json');
+    const resolve = vi.fn();
+
+    worker.pendingRequest = { responseFile, resolve, reject: vi.fn() };
+    worker.handleDone(`${responseFile}\r`);
+
+    expect(resolve).toHaveBeenCalledOnce();
+    expect(worker.pendingRequest).toBeNull();
+  });
+});
+
+describeOrSkip('PythonWorker', () => {
+  let sharedWorker: PythonWorker;
+  let multiApiWorker: PythonWorker;
+  let errorWorker: PythonWorker;
+  let nonexistentFunctionWorker: PythonWorker;
+  let wrongNameWorker: PythonWorker;
+  let embeddingsOnlyWorker: PythonWorker;
+  let loggingWorker: PythonWorker;
+  let protocolCollisionWorker: PythonWorker;
+  let forgedMarkerWorker: PythonWorker;
+  const fixturesDir = path.join(__dirname, 'fixtures');
+  const testScriptPath = path.join(__dirname, 'fixtures', 'simple_provider.py');
+  const multiApiPath = path.join(__dirname, 'fixtures', 'multi_api_provider.py');
+  const errorPath = path.join(__dirname, 'fixtures', 'error_provider.py');
+  const nonexistentPath = path.join(__dirname, 'fixtures', 'test_nonexistent_function.py');
+  const wrongNamePath = path.join(__dirname, 'fixtures', 'test_wrong_function_name.py');
+  const embeddingsOnlyPath = path.join(__dirname, 'fixtures', 'test_embeddings_only.py');
+  const loggingPath = path.join(__dirname, 'fixtures', 'logging_provider.py');
+  const protocolCollisionPath = path.join(__dirname, 'fixtures', 'protocol_collision_provider.py');
+  const forgedMarkerPath = path.join(__dirname, 'fixtures', 'forged_marker_provider.py');
+
+  beforeAll(async () => {
+    // Create test fixture
+    await fs.promises.mkdir(fixturesDir, { recursive: true });
+
+    await fs.promises.writeFile(
+      testScriptPath,
+      `
+def call_api(prompt, options, context):
+    return {"output": f"Echo: {prompt}"}
+`,
+    );
+
+    await fs.promises.writeFile(
+      multiApiPath,
+      `
+def call_api(prompt, options, context):
+    return {"output": f"text: {prompt}", "type": "text"}
+
+def call_embedding_api(prompt, options, context):
+    return {"output": [0.1, 0.2, 0.3], "type": "embedding"}
+
+def call_classification_api(prompt, options, context):
+    return {"output": "positive", "type": "classification"}
+`,
+    );
+
+    await fs.promises.writeFile(
+      errorPath,
+      `
+def call_api(prompt, options, context):
+    if prompt == "error":
+        raise ValueError("Intentional error for testing")
+    return {"output": f"Success: {prompt}"}
+`,
+    );
+
+    // Uses Python's default logging format (LEVEL:name:message) so the test
+    // exercises what real providers emit when they don't customize logging.
+    await fs.promises.writeFile(
+      loggingPath,
+      `
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+def call_api(prompt, options, context):
+    logging.info("provider startup details")
+    logging.warning("provider warning")
+    return {"output": f"Logged: {prompt}"}
+`,
+    );
+
+    await fs.promises.writeFile(
+      protocolCollisionPath,
+      `
+import time
+
+def call_api(prompt, options, context):
+    print("DONE", flush=True)
+    time.sleep(0.02)
+    return {"output": f"Completed: {prompt}"}
+`,
+    );
+
+    // Emits a DONE| line with attacker-controlled content. The wrapper's real
+    // DONE|<response_file> message must still resolve the request, and the
+    // unrelated marker must be ignored without being repeated in logs.
+    await fs.promises.writeFile(
+      forgedMarkerPath,
+      `
+import time
+
+def call_api(prompt, options, context):
+    print("DONE|sensitive-provider-marker", flush=True)
+    time.sleep(0.02)
+    return {"output": f"Completed: {prompt}"}
+`,
+    );
+
+    await Promise.all([
+      fs.promises.access(nonexistentPath),
+      fs.promises.access(wrongNamePath),
+      fs.promises.access(embeddingsOnlyPath),
+    ]);
+
+    sharedWorker = new PythonWorker(testScriptPath, 'call_api');
+    multiApiWorker = new PythonWorker(multiApiPath, 'call_api');
+    errorWorker = new PythonWorker(errorPath, 'call_api');
+    nonexistentFunctionWorker = new PythonWorker(nonexistentPath, 'call_api');
+    wrongNameWorker = new PythonWorker(wrongNamePath, 'call_api');
+    embeddingsOnlyWorker = new PythonWorker(embeddingsOnlyPath, 'call_api');
+    loggingWorker = new PythonWorker(loggingPath, 'call_api');
+    protocolCollisionWorker = new PythonWorker(protocolCollisionPath, 'call_api');
+    forgedMarkerWorker = new PythonWorker(forgedMarkerPath, 'call_api');
+
+    await Promise.all([
+      sharedWorker.initialize(),
+      multiApiWorker.initialize(),
+      errorWorker.initialize(),
+      nonexistentFunctionWorker.initialize(),
+      wrongNameWorker.initialize(),
+      embeddingsOnlyWorker.initialize(),
+      loggingWorker.initialize(),
+      protocolCollisionWorker.initialize(),
+      forgedMarkerWorker.initialize(),
+    ]);
+  });
+
+  afterAll(async () => {
+    await Promise.all(
+      [
+        sharedWorker,
+        multiApiWorker,
+        errorWorker,
+        nonexistentFunctionWorker,
+        wrongNameWorker,
+        embeddingsOnlyWorker,
+        loggingWorker,
+        protocolCollisionWorker,
+        forgedMarkerWorker,
+      ]
+        .filter((worker): worker is PythonWorker => Boolean(worker))
+        .map((worker) => worker.shutdown()),
+    );
+
+    for (const fixturePath of [
+      testScriptPath,
+      multiApiPath,
+      errorPath,
+      loggingPath,
+      protocolCollisionPath,
+      forgedMarkerPath,
+    ]) {
+      if (fs.existsSync(fixturePath)) {
+        fs.unlinkSync(fixturePath);
+      }
+    }
+  });
+
+  it(
+    'should initialize and become ready',
+    async () => {
+      expect(sharedWorker.isReady()).toBe(true);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should execute a function call',
+    async () => {
+      const result = (await sharedWorker.call('call_api', ['Hello world', {}, {}])) as {
+        output: string;
+      };
+      expect(result.output).toBe('Echo: Hello world');
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should reuse the same process for multiple calls',
+    async () => {
+      const result1 = (await sharedWorker.call('call_api', ['First', {}, {}])) as {
+        output: string;
+      };
+      const result2 = (await sharedWorker.call('call_api', ['Second', {}, {}])) as {
+        output: string;
+      };
+
+      expect(result1.output).toBe('Echo: First');
+      expect(result2.output).toBe('Echo: Second');
+      // Same process should be used (we'll verify in implementation)
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should call different function names dynamically per request',
+    async () => {
+      // Call different functions in the same worker
+      const textResult = await multiApiWorker.call('call_api', ['hello', {}, {}]);
+      const embeddingResult = await multiApiWorker.call('call_embedding_api', ['hello', {}, {}]);
+      const classResult = await multiApiWorker.call('call_classification_api', ['hello', {}, {}]);
+
+      // Verify each function was called correctly
+      expect((textResult as Record<string, unknown>).type).toBe('text');
+      expect((textResult as Record<string, unknown>).output).toBe('text: hello');
+
+      expect((embeddingResult as Record<string, unknown>).type).toBe('embedding');
+      expect((embeddingResult as Record<string, unknown>).output).toEqual([0.1, 0.2, 0.3]);
+
+      expect((classResult as Record<string, unknown>).type).toBe('classification');
+      expect((classResult as Record<string, unknown>).output).toBe('positive');
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should not surface routine Python stderr logging as worker errors',
+    async () => {
+      const result = (await loggingWorker.call('call_api', ['hello', {}, {}])) as {
+        output: string;
+      };
+
+      expect(result.output).toBe('Logged: hello');
+      expect(logger.error).not.toHaveBeenCalled();
+      // Default Python logging format is LEVEL:name:message (e.g. WARNING:root:...).
+      expect(logger.warn).toHaveBeenCalledWith(
+        expect.stringContaining('Python worker stderr: WARNING:root:provider warning'),
+      );
+      expect(logger.info).toHaveBeenCalledWith(
+        expect.stringContaining('Python worker stderr: INFO:root:provider startup details'),
+      );
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should not treat provider stdout as a response completion marker',
+    async () => {
+      const result = (await protocolCollisionWorker.call('call_api', ['hello', {}, {}])) as {
+        output: string;
+      };
+      const secondResult = (await protocolCollisionWorker.call('call_api', ['again', {}, {}])) as {
+        output: string;
+      };
+
+      expect(result.output).toBe('Completed: hello');
+      expect(secondResult.output).toBe('Completed: again');
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should ignore forged DONE| markers without logging provider-controlled content',
+    async () => {
+      const result = (await forgedMarkerWorker.call('call_api', ['hello', {}, {}])) as {
+        output: string;
+      };
+      const secondResult = (await forgedMarkerWorker.call('call_api', ['again', {}, {}])) as {
+        output: string;
+      };
+
+      // If the worker accepted the script's forged DONE marker,
+      // executeCall would resolve early, read the empty reserved response
+      // file, and either throw or return undefined.
+      expect(result.output).toBe('Completed: hello');
+      expect(secondResult.output).toBe('Completed: again');
+      expect(logger.debug).toHaveBeenCalledWith(
+        'Python worker ignored DONE marker that did not match the in-flight request',
+        { hasPendingRequest: true },
+      );
+      expect(JSON.stringify(vi.mocked(logger.debug).mock.calls)).not.toContain(
+        'sensitive-provider-marker',
+      );
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle Python errors gracefully',
+    async () => {
+      // Should succeed
+      const goodResult = (await errorWorker.call('call_api', ['good', {}, {}])) as Record<
+        string,
+        unknown
+      >;
+      expect(goodResult.output).toBe('Success: good');
+
+      // Should throw error
+      await expect(errorWorker.call('call_api', ['error', {}, {}])).rejects.toThrow(
+        'Intentional error',
+      );
+
+      // Worker should still be usable after error
+      const afterErrorResult = (await errorWorker.call('call_api', [
+        'still works',
+        {},
+        {},
+      ])) as Record<string, unknown>;
+      expect(afterErrorResult.output).toBe('Success: still works');
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle calling non-existent function gracefully',
+    async () => {
+      // Try to call a function that doesn't exist
+      // This should throw an error about the function not existing, not ENOENT
+      await expect(
+        nonexistentFunctionWorker.call('call_nonexistent_api', ['test', {}]),
+      ).rejects.toThrow(/has no attribute|AttributeError/);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should provide helpful error message with function name suggestions',
+    async () => {
+      // User has 'get_embedding_api' but we're looking for 'call_embedding_api'
+      try {
+        await wrongNameWorker.call('call_embedding_api', ['test', {}]);
+        expect.fail('Should have thrown an error');
+      } catch (error: any) {
+        const errorMessage = error.message;
+
+        // Should include helpful information
+        expect(errorMessage).toContain("Function 'call_embedding_api' not found");
+        expect(errorMessage).toContain('Available functions in your module');
+        expect(errorMessage).toContain('get_embedding_api'); // Shows what they have
+        expect(errorMessage).toContain('Expected function names for promptfoo');
+        expect(errorMessage).toContain('call_api'); // Shows valid options
+        expect(errorMessage).toContain('call_embedding_api');
+        expect(errorMessage).toContain('call_classification_api');
+        expect(errorMessage).toContain('Did you mean to rename'); // Fuzzy match suggestion
+        expect(errorMessage).toContain('promptfoo.dev/docs/providers/python'); // Doc link
+
+        // Should NOT be generic ENOENT error
+        expect(errorMessage).not.toContain('ENOENT');
+        expect(errorMessage).not.toContain('no such file or directory');
+      }
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should support embeddings-only provider without call_api defined',
+    async () => {
+      // Call the embedding function directly
+      const result: any = await embeddingsOnlyWorker.call('call_embedding_api', [
+        'test prompt',
+        {},
+      ]);
+
+      // Should return valid embedding
+      expect(result).toHaveProperty('embedding');
+      expect(Array.isArray(result.embedding)).toBe(true);
+      expect(result.embedding.length).toBeGreaterThan(0);
+    },
+    TEST_TIMEOUT,
+  );
+});

test/python/workerPool.test.ts

@@ -0,0 +1,214 @@
+import fs from 'fs';
+import path from 'path';
+
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { PythonWorkerPool } from '../../src/python/workerPool';
+
+// Windows CI has severe filesystem delays (antivirus, etc.) - allow up to 90s
+// Non-Windows CI can also have timing variance with Python IPC, so use 15s (matching windows-path.test.ts)
+const TEST_TIMEOUT = process.platform === 'win32' ? 90000 : 15000;
+
+// Skip on Windows CI due to aggressive file security policies blocking temp file IPC
+// Works fine on local Windows and all other platforms
+const describeOrSkip = process.platform === 'win32' && process.env.CI ? describe.skip : describe;
+
+describeOrSkip('PythonWorkerPool', () => {
+  let singleWorkerPool: PythonWorkerPool;
+  let multiWorkerPool: PythonWorkerPool;
+  let multiApiPool: PythonWorkerPool;
+  const testScriptPath = path.join(__dirname, 'fixtures', 'counter_provider.py');
+  const multiApiPath = path.join(__dirname, 'fixtures', 'pool_multi_api.py');
+  const fixturesDir = path.join(__dirname, 'fixtures');
+
+  beforeAll(async () => {
+    // Create fixtures directory if it doesn't exist
+    if (!fs.existsSync(fixturesDir)) {
+      fs.mkdirSync(fixturesDir, { recursive: true });
+    }
+
+    // Create test fixture with global state
+    fs.writeFileSync(
+      testScriptPath,
+      `
+# Global counter - persists across calls within same worker
+call_count = 0
+
+def call_api(prompt, options, context):
+    global call_count
+    call_count += 1
+    return {"output": f"Call #{call_count}: {prompt}", "count": call_count}
+
+def reset_call_count():
+    global call_count
+    call_count = 0
+    return {"count": call_count}
+`,
+    );
+
+    fs.writeFileSync(
+      multiApiPath,
+      `
+def call_api(prompt, options, context):
+    return {"output": f"text: {prompt}", "type": "text"}
+
+def call_embedding_api(prompt, options, context):
+    return {"output": [0.1, 0.2], "type": "embedding"}
+`,
+    );
+
+    singleWorkerPool = new PythonWorkerPool(testScriptPath, 'call_api', 1);
+    multiWorkerPool = new PythonWorkerPool(testScriptPath, 'call_api', 2);
+    multiApiPool = new PythonWorkerPool(multiApiPath, 'call_api', 2);
+
+    await Promise.all([
+      singleWorkerPool.initialize(),
+      multiWorkerPool.initialize(),
+      multiApiPool.initialize(),
+    ]);
+  });
+
+  beforeEach(async () => {
+    await Promise.all([
+      singleWorkerPool.execute('reset_call_count', []),
+      ...Array.from({ length: multiWorkerPool.getWorkerCount() }, () =>
+        multiWorkerPool.execute('reset_call_count', []),
+      ),
+    ]);
+  });
+
+  afterAll(async () => {
+    await Promise.all(
+      [singleWorkerPool, multiWorkerPool, multiApiPool]
+        .filter((pool): pool is PythonWorkerPool => Boolean(pool))
+        .map((pool) => pool.shutdown()),
+    );
+
+    for (const fixturePath of [testScriptPath, multiApiPath]) {
+      if (fs.existsSync(fixturePath)) {
+        fs.unlinkSync(fixturePath);
+      }
+    }
+  });
+
+  it(
+    'should initialize pool with specified worker count',
+    async () => {
+      expect(multiWorkerPool.getWorkerCount()).toBe(2);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it('should reject invalid worker counts', async () => {
+    // Test zero workers
+    const zeroWorkerPool = new PythonWorkerPool(testScriptPath, 'call_api', 0);
+    await expect(zeroWorkerPool.initialize()).rejects.toThrow(
+      'Invalid worker count: 0. Must be at least 1.',
+    );
+
+    // Test negative workers
+    const negativeWorkerPool = new PythonWorkerPool(testScriptPath, 'call_api', -1);
+    await expect(negativeWorkerPool.initialize()).rejects.toThrow(
+      'Invalid worker count: -1. Must be at least 1.',
+    );
+  });
+
+  it(
+    'should execute calls sequentially with 1 worker',
+    async () => {
+      const result1 = await singleWorkerPool.execute('call_api', ['First', {}, {}]);
+      const result2 = await singleWorkerPool.execute('call_api', ['Second', {}, {}]);
+      const result3 = await singleWorkerPool.execute('call_api', ['Third', {}, {}]);
+
+      // Same worker, counter increments
+      expect(result1.count).toBe(1);
+      expect(result2.count).toBe(2);
+      expect(result3.count).toBe(3);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle concurrent calls with multiple workers',
+    async () => {
+      // Execute 4 calls concurrently
+      const promises = [
+        multiWorkerPool.execute('call_api', ['Call 1', {}, {}]),
+        multiWorkerPool.execute('call_api', ['Call 2', {}, {}]),
+        multiWorkerPool.execute('call_api', ['Call 3', {}, {}]),
+        multiWorkerPool.execute('call_api', ['Call 4', {}, {}]),
+      ];
+
+      const results = await Promise.all(promises);
+
+      // Each worker maintains its own counter
+      // With 2 workers, work should be distributed across both (not all to one worker)
+      const counts = results.map((r) => r.count);
+      const uniqueCounts = new Set(counts);
+
+      // Verify multiple workers were used (at least 2 different counts)
+      expect(uniqueCounts.size).toBeGreaterThan(1);
+
+      // Verify all calls completed successfully
+      expect(results.length).toBe(4);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should queue requests when all workers busy',
+    async () => {
+      // Start 3 concurrent calls with 1 worker - should queue
+      const promises = [
+        singleWorkerPool.execute('call_api', ['Q1', {}, {}]),
+        singleWorkerPool.execute('call_api', ['Q2', {}, {}]),
+        singleWorkerPool.execute('call_api', ['Q3', {}, {}]),
+      ];
+
+      const results = await Promise.all(promises);
+
+      // All should complete (queued and executed)
+      expect(results.length).toBe(3);
+      expect(results[0].count).toBe(1);
+      expect(results[1].count).toBe(2);
+      expect(results[2].count).toBe(3);
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should handle different function names across pool',
+    async () => {
+      const results = await Promise.all([
+        multiApiPool.execute('call_api', ['hello', {}, {}]),
+        multiApiPool.execute('call_embedding_api', ['world', {}, {}]),
+        multiApiPool.execute('call_api', ['again', {}, {}]),
+      ]);
+
+      expect(results[0].type).toBe('text');
+      expect(results[1].type).toBe('embedding');
+      expect(results[2].type).toBe('text');
+    },
+    TEST_TIMEOUT,
+  );
+
+  it(
+    'should process queued requests after worker becomes available',
+    async () => {
+      // Fire off 5 requests - should queue and process sequentially
+      const promises = [];
+      for (let i = 0; i < 5; i++) {
+        promises.push(singleWorkerPool.execute('call_api', [`request-${i}`, {}, {}]));
+      }
+
+      const results = await Promise.all(promises);
+
+      // All requests should complete
+      expect(results.length).toBe(5);
+
+      // Counter should increment sequentially (all in same worker)
+      const counts = results.map((r) => r.count);
+      expect(counts).toEqual([1, 2, 3, 4, 5]);
+    },
+    TEST_TIMEOUT,
+  );
+});

test/python/wrapper.test.ts

@@ -0,0 +1,213 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+  getSysExecutable,
+  runPython,
+  state,
+  tryPath,
+  validatePythonPath,
+} from '../../src/python/pythonUtils';
+import { runPythonCode } from '../../src/python/wrapper';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../../src/util/secureTempFiles';
+import { mockProcessEnv } from '../util/utils';
+
+vi.mock('../../src/esm');
+vi.mock('python-shell');
+vi.mock('../../src/util/secureTempFiles', () => ({
+  createSecureTempDirectory: vi.fn(),
+  removeSecureTempDirectory: vi.fn(),
+  writeSecureTempFile: vi.fn(),
+}));
+vi.mock('../../src/python/pythonUtils', async () => {
+  const originalModule = await vi.importActual<typeof import('../../src/python/pythonUtils')>(
+    '../../src/python/pythonUtils',
+  );
+  return {
+    ...originalModule,
+    validatePythonPath: vi.fn(),
+    runPython: vi.fn(originalModule.runPython),
+    tryPath: vi.fn(),
+    getSysExecutable: vi.fn(),
+    // Use the real state object so all implementations share the same cache
+    state: originalModule.state,
+  };
+});
+interface TestResult {
+  testId: number;
+  result: string;
+}
+interface MixedTestResult {
+  testId: number;
+  result: string;
+  isExplicit: boolean;
+}
+describe('wrapper', () => {
+  let restoreEnv: () => void;
+
+  beforeAll(() => {
+    restoreEnv = mockProcessEnv({ PROMPTFOO_PYTHON: undefined });
+  });
+
+  afterAll(() => {
+    restoreEnv();
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(createSecureTempDirectory).mockResolvedValue('/tmp/promptfoo-python-code-test');
+    vi.mocked(removeSecureTempDirectory).mockResolvedValue(undefined);
+    vi.mocked(writeSecureTempFile).mockImplementation(
+      async (directory: string, filename: string) => `${directory}/${filename}`,
+    );
+    vi.mocked(validatePythonPath).mockImplementation(
+      (pythonPath: string, _isExplicit: boolean): Promise<string> => {
+        state.cachedPythonPath = pythonPath;
+        return Promise.resolve(pythonPath);
+      },
+    );
+  });
+  describe('runPythonCode', () => {
+    it('should clean up the temporary files after execution', async () => {
+      const mockRunPython = vi.fn().mockResolvedValue('cleanup test');
+      vi.mocked(runPython).mockImplementation(mockRunPython);
+      await runPythonCode('print("cleanup test")', 'main', []);
+      expect(createSecureTempDirectory).toHaveBeenCalledWith('promptfoo-python-code-');
+      expect(writeSecureTempFile).toHaveBeenCalledWith(
+        '/tmp/promptfoo-python-code-test',
+        'script.py',
+        'print("cleanup test")',
+      );
+      expect(mockRunPython).toHaveBeenCalledTimes(1);
+      expect(mockRunPython).toHaveBeenCalledWith(
+        '/tmp/promptfoo-python-code-test/script.py',
+        'main',
+        [],
+      );
+      expect(removeSecureTempDirectory).toHaveBeenCalledWith('/tmp/promptfoo-python-code-test');
+    });
+    it('should execute Python code from a string and read the output file', async () => {
+      const mockOutput = { type: 'final_result', data: 'execution result' };
+      const mockRunPython = vi.mocked(runPython);
+      mockRunPython.mockResolvedValue(mockOutput.data);
+      const code = 'print("Hello, world!")';
+      const result = await runPythonCode(code, 'main', []);
+      expect(result).toBe('execution result');
+      expect(mockRunPython).toHaveBeenCalledWith(expect.stringContaining('.py'), 'main', []);
+      expect(writeSecureTempFile).toHaveBeenCalledWith(
+        '/tmp/promptfoo-python-code-test',
+        'script.py',
+        code,
+      );
+    });
+  });
+  describe('validatePythonPath race conditions', () => {
+    beforeAll(async () => {
+      // Restore the real validatePythonPath implementation while keeping
+      // tryPath and getSysExecutable mocked to avoid actual system calls
+      const realModule = await vi.importActual<typeof import('../../src/python/pythonUtils')>(
+        '../../src/python/pythonUtils',
+      );
+      vi.mocked(validatePythonPath).mockImplementation(realModule.validatePythonPath);
+    });
+
+    beforeEach(() => {
+      // Reset the cached path and validation promise before each test
+      state.cachedPythonPath = null;
+      state.validationPromise = null;
+
+      // Yield once before resolving so concurrent callers can race onto the
+      // shared validation promise — this is the same race condition the test
+      // is exercising; wall-clock delay isn't needed.
+      vi.mocked(tryPath).mockImplementation(async (_path: string) => {
+        await Promise.resolve();
+        return '/usr/bin/python3';
+      });
+      vi.mocked(getSysExecutable).mockImplementation(async () => {
+        await Promise.resolve();
+        return '/usr/bin/python3';
+      });
+    });
+    it('should handle concurrent validatePythonPath calls without race conditions', async () => {
+      // Launch multiple concurrent validations
+      const concurrentCalls = 10;
+      const promises = Array.from({ length: concurrentCalls }, (_, i) =>
+        validatePythonPath('python', false).then(
+          (result: string): TestResult => ({ testId: i, result }),
+        ),
+      );
+      const results = await Promise.all(promises);
+      // All calls should succeed
+      expect(results).toHaveLength(concurrentCalls);
+      results.forEach((result: TestResult) => {
+        expect(result.result).toBeTruthy();
+        expect(typeof result.result).toBe('string');
+      });
+      // All results should be consistent (no race condition)
+      // If there was a race condition, different calls might get different results
+      const uniqueResults = new Set(results.map((r: TestResult) => r.result));
+      expect(uniqueResults.size).toBe(1);
+      // Cache should be populated
+      expect(state.cachedPythonPath).toBeTruthy();
+    }, 10000); // Increase timeout for this test
+    it('should handle mixed explicit/implicit validation calls consistently', async () => {
+      // Create mixed explicit/implicit calls
+      const mixedPromises = Array.from({ length: 8 }, (_, i) => {
+        const isExplicit = i % 2 === 0;
+        return validatePythonPath('python', isExplicit).then(
+          (result: string): MixedTestResult => ({
+            testId: i,
+            result,
+            isExplicit,
+          }),
+        );
+      });
+      const results = await Promise.all(mixedPromises);
+      // All calls should succeed
+      expect(results).toHaveLength(8);
+      results.forEach((result: MixedTestResult) => {
+        expect(result.result).toBeTruthy();
+        expect(typeof result.result).toBe('string');
+      });
+      // Check consistency between explicit and implicit results
+      const explicitResults = results
+        .filter((r: MixedTestResult) => r.isExplicit)
+        .map((r: MixedTestResult) => r.result);
+      const implicitResults = results
+        .filter((r: MixedTestResult) => !r.isExplicit)
+        .map((r: MixedTestResult) => r.result);
+      const uniqueExplicitResults = new Set(explicitResults);
+      const uniqueImplicitResults = new Set(implicitResults);
+      // Both explicit and implicit calls should return consistent results
+      // If there was a race condition, explicit and implicit calls might return different results
+      expect(uniqueExplicitResults.size).toBe(1);
+      expect(uniqueImplicitResults.size).toBe(1);
+      // Explicit and implicit results should be the same
+      expect(explicitResults[0]).toBe(implicitResults[0]);
+    }, 10000);
+    it('should handle rapid successive calls without race conditions', async () => {
+      // Launch rapid successive calls
+      const rapidCalls = 20;
+      const promises = Array.from({ length: rapidCalls }, (_, i) =>
+        validatePythonPath('python', false).then(
+          (result: string): TestResult => ({ testId: i, result }),
+        ),
+      );
+      const results = await Promise.all(promises);
+      // All calls should succeed
+      expect(results).toHaveLength(rapidCalls);
+      results.forEach((result: TestResult) => {
+        expect(result.result).toBeTruthy();
+        expect(typeof result.result).toBe('string');
+      });
+      // All results should be consistent
+      // If there was a race condition, different calls could get different cached values
+      const uniqueResults = new Set(results.map((r: TestResult) => r.result));
+      expect(uniqueResults.size).toBe(1);
+      // Cache should be populated with the consistent result
+      expect(state.cachedPythonPath).toBe(results[0].result);
+    }, 10000);
+  });
+});